
import io
import json
import numpy as np
import os
import pandas as pd
import shutil
import struct
import sys
import thriftpy
import warnings

import numba

from thriftpy.protocol.compact import TCompactProtocolFactory
from thriftpy.protocol.exc import TProtocolException
from .thrift_filetransport import TFileTransport
from .thrift_structures import parquet_thrift
from .compression import compress_data, decompress_data
from . import encoding
from .util import default_openw, default_mkdirs, sep_from_open, ParquetException

MARKER = b'PAR1'
NaT = np.timedelta64(None).tobytes()  # require numpy version >= 1.7
nat = np.datetime64('NaT').view('int64')

typemap = {  # primitive type, converted type, bit width
    'bool': (parquet_thrift.Type.BOOLEAN, None, 1),
    'int32': (parquet_thrift.Type.INT32, None, 32),
    'int64': (parquet_thrift.Type.INT64, None, 64),
    'int8': (parquet_thrift.Type.INT32, parquet_thrift.ConvertedType.INT_8, 8),
    'int16': (parquet_thrift.Type.INT32, parquet_thrift.ConvertedType.INT_16, 16),
    'uint8': (parquet_thrift.Type.INT32, parquet_thrift.ConvertedType.UINT_8, 8),
    'uint16': (parquet_thrift.Type.INT32, parquet_thrift.ConvertedType.UINT_16, 16),
    'uint32': (parquet_thrift.Type.INT32, parquet_thrift.ConvertedType.UINT_32, 32),
    'float32': (parquet_thrift.Type.FLOAT, None, 32),
    'float64': (parquet_thrift.Type.DOUBLE, None, 64),
    'float16': (parquet_thrift.Type.FLOAT, None, 16),
}

revmap = {parquet_thrift.Type.INT32: np.int32,
          parquet_thrift.Type.INT64: np.int64,
          parquet_thrift.Type.FLOAT: np.float32}


def find_type(data, convert=False, fixed_text=None):
    """ Get appropriate typecodes for column dtype

    Data conversion may happen here, only at write time.

    The user is expected to transform their data into the appropriate dtype
    before saving to parquet, we will not make any assumptions for them.

    If the dtype is "object" the first ten items will be examined, and is str
    or bytes, will be stored as variable or fixed length byte strings;
    if dict or list, (nested data) will be stored with JSON encoding.

    In the case of catagoricals, the data type refers to the labels; the data
    (codes) will be stored as int. The labels are usually variable length
    strings.

    BOOLs will be bitpacked using np.packbits. To instead keep the default numpy
    representation of one byte per value, change to int8 or uint8 type

    Known types that cannot be represented (must be first converted another
    type or to raw binary): float128, complex

    Parameters
    ----------
    data: pd.Series
    convert: bool (False)
        Whether to actually perform the conversion, or just infer types
    fixed_text: int or None
        For str and bytes, the fixed-string length to use. If None, object
        column will remain variable length.

    Returns
    -------
    - a thrift schema element
    - a thrift typecode to be passed to the column chunk writer
    - converted data (None if convert is False)

    """
    out = None
    dtype = data.dtype
    if dtype.name in typemap:
        type, converted_type, width = typemap[dtype.name]
        if type in revmap and convert:
            out = data.values.astype(revmap[type], copy=False)
        elif type == parquet_thrift.Type.BOOLEAN and convert:
            padded = np.lib.pad(data.values, (0, 8 - (len(data) % 8)),
                                'constant', constant_values=(0, 0))
            out = np.packbits(padded.reshape(-1, 8)[:, ::-1].ravel())
        elif convert:
            out = data.values
    elif "S" in str(dtype)[:2] or "U" in str(dtype)[:2]:
        # TODO: check effect of unicode
        type, converted_type, width = (parquet_thrift.Type.FIXED_LEN_BYTE_ARRAY,
                                       None, dtype.itemsize)
        if convert:
            out = data.values
    elif dtype == "O":
        head = data[:10] if isinstance(data, pd.Index) else data.valid()[:10]
        if all(isinstance(i, str) for i in head):
            if fixed_text:
                width = fixed_text
                if convert:
                    out = data.str.encode('utf8').values.astype('S%i' % width)
                type, converted_type = (
                    parquet_thrift.Type.FIXED_LEN_BYTE_ARRAY,
                    parquet_thrift.ConvertedType.UTF8)
            else:
                type, converted_type, width = (parquet_thrift.Type.BYTE_ARRAY,
                                               parquet_thrift.ConvertedType.UTF8, None)
                if convert:
                    out = data.str.encode('utf8').values
        elif all(isinstance(i, bytes) for i in head):
            if fixed_text:
                width = fixed_text
                if convert:
                    out = data.values.astype('S%i' % width)
                type, converted_type = (
                    parquet_thrift.Type.FIXED_LEN_BYTE_ARRAY, None)
            else:
                type, converted_type, width = parquet_thrift.Type.BYTE_ARRAY, None, None
                if convert:
                    out = data.values
        elif all(isinstance(i, (list, dict)) for i in head):
            type, converted_type, width = (parquet_thrift.Type.BYTE_ARRAY,
                                           parquet_thrift.ConvertedType.JSON, None)
            if convert:
                out = data.map(json.dumps).str.encode('utf8').values
        else:
            raise ValueError("Data type conversion unknown: %s" % dtype)
    elif str(dtype).startswith("datetime64"):
        type, converted_type, width = (parquet_thrift.Type.INT64,
                                       parquet_thrift.ConvertedType.TIMESTAMP_MICROS, None)
        if hasattr(dtype, 'tz') and str(dtype.tz) != 'UTC':
            warnings.warn('Coercing datetimes to UTC')
        if convert:
            out = np.empty(len(data), 'int64')
            time_shift(data.values.view('int64'), out)
    elif str(dtype).startswith("timedelta64"):
        type, converted_type, width = (parquet_thrift.Type.INT64,
                                       parquet_thrift.ConvertedType.TIME_MICROS, None)
        if convert:
            out = np.empty(len(data), 'int64')
            time_shift(data.values.view('int64'), out)
    else:
        raise ValueError("Don't know how to convert data type: %s" % dtype)
    # TODO: pandas has no explicit support for Decimal
    se = parquet_thrift.SchemaElement(name=data.name, type_length=width,
                                      converted_type=converted_type, type=type,
                                      repetition_type=parquet_thrift.FieldRepetitionType.REQUIRED)
    return se, type, out


@numba.njit()
def time_shift(indata, outdata, factor=1000):
    for i in range(len(indata)):
        if indata[i] == nat:
            outdata[i] = nat
        else:
            outdata[i] = indata[i] // factor


def thrift_print(structure, offset=0):
    """
    Handy recursive text ouput for thrift structures
    """
    if not isinstance(structure, thriftpy.thrift.TPayload):
        return str(structure)
    s = str(structure.__class__) + '\n'
    for key in dir(structure):
        if key.startswith('_') or key in ['thrift_spec', 'read', 'write',
                                          'default_spec']:
            continue
        s = s + ' ' * offset + key + ': ' + thrift_print(getattr(structure, key)
                                                         , offset+2) + '\n'
    return s
thriftpy.thrift.TPayload.__str__ = thrift_print
thriftpy.thrift.TPayload.__repr__ = thrift_print


def write_thrift(fobj, thrift):
    """Write binary compact representation of thiftpy structured object

    Parameters
    ----------
    fobj: open file-like object (binary mode)
    thrift: thriftpy object to write

    Returns
    -------
    Number of bytes written
    """
    t0 = fobj.tell()
    tout = TFileTransport(fobj)
    pout = TCompactProtocolFactory().get_protocol(tout)
    try:
        thrift.write(pout)
        fail = False
    except TProtocolException as e:
        typ, val, tb = sys.exc_info()
        frames = []
        while tb is not None:
            frames.append(tb)
            tb = tb.tb_next
        frame = [tb for tb in frames if 'write_struct' in str(tb.tb_frame.f_code)]
        variables = frame[0].tb_frame.f_locals
        obj = variables['obj']
        name = variables['fname']
        fail = True
    if fail:
        raise ParquetException('Thrift parameter validation failure %s'
                               ' when writing: %s-> Field: %s' % (
            val.args[0], obj, name
        ))
    return fobj.tell() - t0


def encode_plain(data, se, fixed_text=None):
    """PLAIN encoding; returns byte representation"""
    se, type, out = find_type(data, True, fixed_text=fixed_text)
    if se.type == parquet_thrift.Type.BYTE_ARRAY:
        return b''.join([struct.pack('<l', len(x)) + x for x in out])
    else:
        return out.tobytes()


@numba.njit(nogil=True)
def encode_unsigned_varint(x, o):
    while x > 127:
        o.write_byte((x & 0x7F) | 0x80)
        x >>= 7
    o.write_byte(x)


@numba.jit(nogil=True)
def zigzag(n):
    " 32-bit only "
    return (n << 1) ^ (n >> 31)


@numba.njit(nogil=True)
def encode_bitpacked_inv(values, width, o):
    bit = 16 - width
    right_byte_mask = 0b11111111
    left_byte_mask = right_byte_mask << 8
    bits = 0
    for v in values:
        bits |= v << bit
        while bit <= 8:
            o.write_byte((bits & left_byte_mask) >> 8)
            bit += 8
            bits = (bits & right_byte_mask) << 8
        bit -= width
    if bit:
        o.write_byte((bits & left_byte_mask) >> 8)


@numba.njit(nogil=True)
def encode_bitpacked(values, width, o):
    """
    Write values packed into width-bits each (which can be >8)

    values is a NumbaIO array (int32)
    o is a NumbaIO output array (uint8), size=(len(values)*width)/8, rounded up.
    """
    bit_packed_count = (len(values) + 7) // 8
    encode_unsigned_varint(bit_packed_count << 1 | 1, o)  # write run header

    bit = 0
    right_byte_mask = 0b11111111
    bits = 0
    for v in values:
        bits |= v << bit
        bit += width
        while bit >= 8:
            o.write_byte(bits & right_byte_mask)
            bit -= 8
            bits >>= 8
    if bit:
        o.write_byte(bits)


def write_length(l, o):
    """ Put a 32-bit length into four bytes in o

    Equivalent to struct.pack('<i', l), but suitable for numba-jit
    """
    right_byte_mask = 0b11111111
    for _ in range(4):
        o.write_byte(l & right_byte_mask)
        l >>= 8


def encode_rle_bp(data, width, o, withlength=False):
    """Write data into o using RLE/bitpacked hybrid

    data : values to encode (int32)
    width : bits-per-value, set by max(data)
    o : output encoding.Numpy8
    withlength : bool
        If definitions/repetitions, length of data must be pre-written
    """
    if withlength:
        start = o.loc
        o.loc = o.loc + 4
    if True:
        # I don't know how one would choose between RLE and bitpack
        encode_bitpacked(data, width, o)
    if withlength:
        end = o.loc
        o.loc = start
        write_length(wnd - start, o)
        o.loc = end


def encode_dict(data, se, _):
    """ The data part of dictionary encoding is always int32s, with RLE/bitpack
    """
    # TODO: should width be a parameter equal to len(cats) ?
    width = encoding.width_from_max_int(data.max())
    ldata = ((len(data) + 7) // 8) * width + 11
    i = data.values.astype(np.int32, copy=False)
    out = encoding.Numpy8(np.empty(ldata, dtype=np.uint8))
    out.write_byte(width)
    encode_rle_bp(i, width, out)
    return out.so_far().tostring()

encode = {
    'PLAIN': encode_plain,
    'RLE': encode_rle_bp,
    'PLAIN_DICTIONARY': encode_dict,
    # 'DELTA_BINARY_PACKED': encode_delta
}


def make_definitions(data, no_nulls):
    """For data that can contain NULLs, produce definition levels binary
    data: either bitpacked bools, or (if number of nulls == 0), single RLE
    block."""
    temp = encoding.Numpy8(np.empty(10, dtype=np.uint8))

    if no_nulls:
        # no nulls at all
        l = len(data)
        encode_unsigned_varint(l << 1, temp)
        temp.write_byte(1)
        block = struct.pack('<i', temp.loc) + temp.so_far().tostring()
        out = data
    else:
        # bitpack bools
        out = encode_plain(data.notnull(), None)

        encode_unsigned_varint(len(out) << 1 | 1, temp)
        head = temp.so_far().tostring()

        block = struct.pack('<i', len(head + out)) + head + out
        out = data.valid()  # better, data[data.notnull()], from above ?
    return block, out


def write_column(f, data, selement, encoding='PLAIN', compression=None):
    """
    Write a single column of data to an open Parquet file

    Parameters
    ----------
    f: open binary file
    data: pandas Series or numpy (1d) array
    selement: thrift SchemaElement
        produced by ``find_type``
    encoding: one of ``parquet_thift.Encoding``
        if the dtype is categorical, this is ignored and dictionary encoding
        automatically used
    compression: str or None
        if not None, must be one of the keys in ``compression.compress``

    Returns
    -------
    chunk: ColumnChunk structure

    """
    has_nulls = selement.repetition_type == parquet_thrift.FieldRepetitionType.OPTIONAL
    fixed_text = selement.type_length
    tot_rows = len(data)

    if has_nulls:
        num_nulls = data.count() - len(data)
        definition_data, data = make_definitions(data, num_nulls == 0)
    else:
        definition_data = b""
        num_nulls = 0

    # No nested field handling (encode those as J/BSON)
    repetition_data = b""

    cats = False
    name = data.name
    diff = 0

    if str(data.dtype) == 'category':
        dph = parquet_thrift.DictionaryPageHeader(
                num_values=len(data.cat.categories),
                encoding=parquet_thrift.Encoding.PLAIN)
        bdata = encode['PLAIN'](pd.Series(data.cat.categories), selement,
                                fixed_text=fixed_text)
        l0 = len(bdata)
        if compression:
            bdata = compress_data(bdata, compression)
            l1 = len(bdata)
        else:
            l1 = l0
        diff += l0 - l1
        ph = parquet_thrift.PageHeader(
                type=parquet_thrift.PageType.DICTIONARY_PAGE,
                uncompressed_page_size=l0, compressed_page_size=l1,
                dictionary_page_header=dph, crc=None)

        dict_start = f.tell()
        write_thrift(f, ph)
        f.write(bdata)
        try:
            max, min = data.max(), data.min()
            max = encode['PLAIN'](pd.Series([max]), selement,
                                  fixed_text=fixed_text)
            min = encode['PLAIN'](pd.Series([min]), selement,
                                  fixed_text=fixed_text)
        except TypeError:
            max, min = None, None
        data = data.cat.codes.astype(np.int32)
        cats = True
        encoding = "PLAIN_DICTIONARY"

    start = f.tell()
    bdata = definition_data + repetition_data + encode[encoding](data, selement,
                                                                 fixed_text)
    try:
        if encoding != 'PLAIN_DICTIONARY':
            max, min = data.max(), data.min()
            max = encode['PLAIN'](pd.Series([max], dtype=data.dtype), selement,
                                  fixed_text=fixed_text)
            min = encode['PLAIN'](pd.Series([min], dtype=data.dtype), selement,
                                  fixed_text=fixed_text)
    except TypeError:
        max, min = None, None

    dph = parquet_thrift.DataPageHeader(
            num_values=tot_rows,
            encoding=getattr(parquet_thrift.Encoding, encoding),
            definition_level_encoding=parquet_thrift.Encoding.RLE,
            repetition_level_encoding=parquet_thrift.Encoding.BIT_PACKED)
    l0 = len(bdata)

    if compression:
        bdata = compress_data(bdata, compression)
        l1 = len(bdata)
    else:
        l1 = l0
    diff += l0 - l1

    ph = parquet_thrift.PageHeader(type=parquet_thrift.PageType.DATA_PAGE,
                                   uncompressed_page_size=l0,
                                   compressed_page_size=l1,
                                   data_page_header=dph, crc=None)

    write_thrift(f, ph)
    f.write(bdata)

    compressed_size = f.tell() - start
    uncompressed_size = compressed_size + diff

    offset = f.tell()
    s = parquet_thrift.Statistics(max=max, min=min, null_count=num_nulls)

    p = [parquet_thrift.PageEncodingStats(
            page_type=parquet_thrift.PageType.DATA_PAGE,
            encoding=parquet_thrift.Encoding.PLAIN, count=1)]

    cmd = parquet_thrift.ColumnMetaData(
            type=selement.type, path_in_schema=[name],
            encodings=[parquet_thrift.Encoding.RLE,
                       parquet_thrift.Encoding.BIT_PACKED,
                       parquet_thrift.Encoding.PLAIN],
            codec=getattr(parquet_thrift.CompressionCodec, compression.upper()) if compression else 0,
            num_values=tot_rows,
            statistics=s,
            data_page_offset=start,
            encoding_stats=p,
            key_value_metadata=[],
            total_uncompressed_size=uncompressed_size,
            total_compressed_size=compressed_size)
    if cats:
        p.append(parquet_thrift.PageEncodingStats(
                page_type=parquet_thrift.PageType.DICTIONARY_PAGE,
                encoding=parquet_thrift.Encoding.PLAIN, count=1))
        cmd.dictionary_page_offset = dict_start
    chunk = parquet_thrift.ColumnChunk(file_offset=offset,
                                       meta_data=cmd)
    write_thrift(f, chunk)
    return chunk


def make_row_group(f, data, schema, file_path=None, compression=None,
                   encoding='PLAIN'):
    """ Make a single row group of a Parquet file """
    rows = len(data)
    if rows == 0:
        return
    rg = parquet_thrift.RowGroup(num_rows=rows, total_byte_size=0, columns=[])

    for column in schema:
        if column.type is not None:
            if isinstance(compression, dict):
                comp = compression.get(column.name, None)
            else:
                comp = compression
            chunk = write_column(f, data[column.name], column,
                                 compression=comp, encoding=encoding)
            rg.columns.append(chunk)
    rg.total_byte_size = sum([c.meta_data.total_uncompressed_size for c in
                              rg.columns])
    return rg


def make_part_file(f, data, schema, compression=None, encoding='PLAIN',
                   fixed_text=None):
    if len(data) == 0:
        return
    with f as f:
        f.write(MARKER)
        rg = make_row_group(f, data, schema, compression=compression,
                            encoding=encoding)
        fmd = parquet_thrift.FileMetaData(num_rows=len(data),
                                          schema=schema,
                                          version=1,
                                          created_by='parquet-python',
                                          row_groups=[rg])
        foot_size = write_thrift(f, fmd)
        f.write(struct.pack(b"<i", foot_size))
        f.write(MARKER)
    return rg


def make_metadata(data, has_nulls=True, ignore_columns=[], fixed_text=None):
    root = parquet_thrift.SchemaElement(name='schema',
                                        num_children=0)

    fmd = parquet_thrift.FileMetaData(num_rows=len(data),
                                      schema=[root],
                                      version=1,
                                      created_by='fastparquet-python',
                                      row_groups=[])

    for column in data.columns:
        if column in ignore_columns:
            continue
        fixed = None if fixed_text is None else fixed_text.get(column, None)
        if str(data[column].dtype) == 'category':
            se, type, _ = find_type(data[column].cat.categories,
                                    fixed_text=fixed)
            se.name = column
        else:
            se, type, _ = find_type(data[column], fixed_text=fixed)
        if has_nulls is None:
            se.repetition_type = type == parquet_thrift.Type.BYTE_ARRAY
        else:
            has_nulls = (has_nulls if has_nulls in [True, False]
                         else column in has_nulls)
        if has_nulls and data[column].dtype.kind != 'i':
            se.repetition_type = parquet_thrift.FieldRepetitionType.OPTIONAL
        fmd.schema.append(se)
        root.num_children += 1
    return fmd


def write(filename, data, row_group_offsets=50000000, encoding="PLAIN",
          compression=None, file_scheme='simple', open_with=default_openw,
          mkdirs=default_mkdirs, has_nulls=None, write_index=None,
          partition_on=[], fixed_text=None):
    """ Write Pandas DataFrame to filename as Parquet Format

    Parameters
    ----------
    filename: string
        File contains everything (if file_scheme='same'), else contains the
        metadata only
    data: pandas dataframe
        The table to write
    row_group_offsets: int or list of ints
        In int, row-groups will be approximately this many rows, rounded down
        to make row groups about the same size; if a list, the explicit index
        values to start new row groups.
    encoding: single value from parquet_thrift.Encoding, if applied to all
        columns, or dict of name:parquet_thrift.Encoding for a different
        encoding per column.
    compression: str, dict
        compression to apply to each column, e.g. GZIP or SNAPPY
    file_scheme: 'simple'|'hive'
        If simple: all goes in a single file
        If hive: each row group is in a separate file, and filename contains
        only the metadata
    open_with: function
        When called with a path/URL, returns an open file-like object
    mkdirs: function
        When called with a path/URL, creates any necessary dictionaries to
        make that location writable, e.g., ``os.makedirs``. This is not
        necessary if using the simple file scheme
    has_nulls: None, bool or list of strings
        Whether columns can have nulls. If a list of strings, those given
        columns will be marked as "optional" in the metadata, and include
        null definition blocks on disk. Some data types (floats and times)
        can instead use the sentinel values NaN and NaT, which are not the same
        as NULL in parquet, but functionally act the same in many cases,
        particularly if converting back to pandas later. A value of None
        will assume nulls for object columns and not otherwise.
    write_index: boolean
        Whether or not to write the index to a separate column.  By default we
        write the index *if* it is not 0, 1, ..., n.
    partition_on: list of column names
        Passed to groupby in order to split data within each row-group,
        producing a structured directory tree. Note: as with pandas, null
        values will be dropped. Ignored if file_scheme is simple.
    fixed_text: {column: int length} or None
        For bytes or str columns, values will be converted
        to fixed-length strings of the given length for the given columns
        before writing, potentially providing a large speed
        boost.

    Examples
    --------
    >>> fastparquet.write('myfile.parquet', df)  # doctest: +SKIP
    """
    sep = sep_from_open(open_with)
    if file_scheme != 'simple' or isinstance(data, pd.core.groupby.DataFrameGroupBy):
        mkdirs(filename)
        fn = sep.join([filename, '_metadata'])
    else:
        fn = filename
    if isinstance(row_group_offsets, int):
        l = len(data)
        nparts = (l - 1) // row_group_offsets + 1
        chunksize = (l - 1) // nparts + 1
        row_group_offsets = list(range(0, l, chunksize))
    with open_with(fn) as f:
        f.write(MARKER)

        if write_index or write_index is None and not (
                isinstance(data.index, pd.RangeIndex) and
                data.index._start == 0 and
                data.index._stop == len(data.index) and
                data.index._step == 1 and data.index.name is None):
            data = data.reset_index()
        ignore = partition_on if file_scheme != 'simple' else []
        fmd = make_metadata(data, has_nulls=has_nulls, ignore_columns=ignore,
                            fixed_text=fixed_text)
        for i, start in enumerate(row_group_offsets):
            end = row_group_offsets[i+1] if i < (len(row_group_offsets) - 1) else None
            if file_scheme == 'simple':
                rg = make_row_group(f, data[start:end], fmd.schema,
                                    compression=compression, encoding=encoding)
            else:
                part = 'part.%i.parquet' % i
                if partition_on:
                    partition_on_columns(
                        data[start:end], partition_on, filename, part, fmd,
                        sep, compression, encoding, open_with, mkdirs
                    )
                    rg = None
                else:
                    partname = sep.join([filename, part])
                    with open_with(partname) as f2:
                        rg = make_part_file(f2, data[start:end], fmd.schema,
                                            compression=compression,
                                            encoding=encoding)
                    for chunk in rg.columns:
                        chunk.file_path = part

            if rg is not None:
                fmd.row_groups.append(rg)

        foot_size = write_thrift(f, fmd)
        f.write(struct.pack(b"<i", foot_size))
        f.write(MARKER)

    if file_scheme != 'simple':
        write_common_metadata(sep.join([filename, '_common_metadata']), fmd,
                              open_with)


def partition_on_columns(data, columns, root_path, partname, fmd, sep,
                         compression, encoding, open_with, mkdirs):
    """ Split each row-group by the given columns

    Each combination of column values (determined by pandas groupby) will
    be written in structured directories.
    """
    gb = data.groupby(columns)
    remaining = list(data)
    for column in columns:
        remaining.remove(column)
    for key in gb.indices:
        df = gb.get_group(key)[remaining]
        path = sep.join(["%s=%s" % (name, val)
                         for name, val in zip(columns, key)])
        relname = sep.join([path, partname])
        mkdirs(root_path + sep + path)
        fullname = sep.join([root_path, path, partname])
        with open_with(fullname) as f2:
            rg = make_part_file(f2, df, fmd.schema,
                                compression=compression,
                                encoding=encoding)
        if rg is not None:
            for chunk in rg.columns:
                chunk.file_path = relname
            fmd.row_groups.append(rg)


def write_common_metadata(fn, fmd, open_with=default_openw):
    """For hive-style parquet, write schema in special shared file"""
    if isinstance(fn, str):
        f = open_with(fn)
    else:
        f = fn
    f.write(MARKER)
    fmd.row_groups = []
    foot_size = write_thrift(f, fmd)
    f.write(struct.pack(b"<i", foot_size))
    f.write(MARKER)
    f.close()
