pywws.DataStore — pywws 13.06_r1023 documentation

### Navigation

- index
- modules |
- pywws 13.06_r1023 documentation »
- Code du module »

# Code source de pywws.DataStore

    # pywws - Python software for USB Wireless Weather Stations
    # http://github.com/jim-easterbrook/pywws
    # Copyright (C) 2008-13  Jim Easterbrook  jim@jim-easterbrook.me.uk

    # This program is free software; you can redistribute it and/or
    # modify it under the terms of the GNU General Public License
    # as published by the Free Software Foundation; either version 2
    # of the License, or (at your option) any later version.

    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    # GNU General Public License for more details.

    # You should have received a copy of the GNU General Public License
    # along with this program; if not, write to the Free Software
    # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

    """
    DataStore.py - stores readings in easy to access files

    A separate file is used for each day's data, to keep memory load
    reasonable. One day at a time is held in memory, and saved to disc
    when another day needs to be accessed.

    Data is accessed in a cross between dictionary and list behaviour.
    The following are all valid:
    # get value nearest 9:30 on Christmas day
    data[data.nearest(datetime(2008, 12, 25, 9, 30))]
    # get entire array, equivalent to data[:] or just data
    data[datetime.min:datetime.max]
    # get last 12 hours of data
    data[datetime.utcnow() - timedelta(hours=12):]
    """

    from ConfigParser import RawConfigParser
    import csv
    from datetime import date, datetime, timedelta
    import os
    import sys
    import time

    def safestrptime(date_string, format=None):
        # time.strptime is time consuming (because it's so flexible?) so don't use
        # it for the fixed format datetime strings in our csv files
        if format:
            return datetime(*(time.strptime(date_string, format)[0:6]))
        return datetime(*map(int, (date_string[0:4],
                                   date_string[5:7],
                                   date_string[8:10],
                                   date_string[11:13],
                                   date_string[14:16],
                                   date_string[17:19])))

    class ParamStore(object):
        def __init__(self, root_dir, file_name):
            if not os.path.isdir(root_dir):
                os.makedirs(root_dir)
            self._path = os.path.join(root_dir, file_name)
            self._dirty = False
            # open config file
            self._config = RawConfigParser()
            self._config.read(self._path)

        def __del__(self):
            self.flush()

        def flush(self):
            if not self._dirty:
                return
            self._dirty = False
            of = open(self._path, 'w')
            self._config.write(of)
            of.close()

        def get(self, section, option, default=None):
            """Get a parameter value and return a string.

            If default is specified and section or option are not defined
            in the file, they are created and set to default, which is
            then the return value.

            """
            if not self._config.has_option(section, option):
                if default is not None:
                    self.set(section, option, default)
                return default
            return self._config.get(section, option)

        def get_datetime(self, section, option, default=None):
            result = self.get(section, option, default)
            if result:
                return safestrptime(result)
            return result

        def set(self, section, option, value):
            """Set option in section to string value."""
            if not self._config.has_section(section):
                self._config.add_section(section)
            elif (self._config.has_option(section, option) and
                  self._config.get(section, option) == value):
                return
            self._config.set(section, option, value)
            self._dirty = True

        def unset(self, section, option):
            """Remove option from section."""
            if not self._config.has_section(section):
                return
            if self._config.has_option(section, option):
                self._config.remove_option(section, option)
                self._dirty = True
            if not self._config.options(section):
                self._config.remove_section(section)
                self._dirty = True

    class params(ParamStore):
        def __init__(self, root_dir):
            """Parameters are stored in a file "weather.ini" in root_dir."""
            ParamStore.__init__(self, root_dir, 'weather.ini')

    class status(ParamStore):
        def __init__(self, root_dir):
            """Status is stored in a file "status.ini" in root_dir."""
            ParamStore.__init__(self, root_dir, 'status.ini')

    class core_store(object):
        def __init__(self, root_dir):
            self._root_dir = root_dir
            self._one_day = timedelta(days=1)
            # initialise cache
            self._cache = []
            self._cache_ptr = 0
            self._cache_lo = date.max.toordinal()
            self._cache_hi = date.min.toordinal()
            self._cache_dirty = False
            # get conservative first and last days for which data (might) exist
            self._fst_day = date.max.toordinal() - 500
            self._lst_day = date.min.toordinal() + 500
            for root, dirs, files in os.walk(self._root_dir):
                files.sort()
                for file in files:
                    if file[0] == '.':
                        continue
                    path, lo, hi = self._get_cache_path(
                        safestrptime(file, "%Y-%m-%d.txt").date())
                    self._fst_day = lo
                    break
                else:
                    dirs.sort()
                    continue
                break
            for root, dirs, files in os.walk(self._root_dir):
                files.sort()
                files.reverse()
                for file in files:
                    if file[0] == '.':
                        continue
                    path, lo, hi = self._get_cache_path(
                        safestrptime(file, "%Y-%m-%d.txt").date())
                    self._lst_day = hi
                    break
                else:
                    dirs.sort()
                    dirs.reverse()
                    continue
                break
        def __del__(self):
            self.flush()
        def _slice(self, i):
            if i.step != None:
                raise TypeError("slice step not permitted")
            a = i.start
            if a == None:
                a = datetime.fromordinal(self._fst_day)
            elif not isinstance(a, datetime):
                raise TypeError("slice indices must be %s or None" % (datetime))
            elif a.toordinal() < self._fst_day:
                a = datetime.fromordinal(self._fst_day)
            b = i.stop
            if b == None:
                b = datetime.max
                lst_day = self._lst_day
            elif not isinstance(b, datetime):
                raise TypeError("slice indices must be %s or None" % (datetime))
            else:
                lst_day = min(b.toordinal() + 1, self._lst_day)
            return a, b, lst_day
        def _get_slice(self, i):
            a, b, lst_day = self._slice(i)
            # go to start of slice
            self._set_cache_ptr(a)
            cache = self._cache
            cache_hi = self._cache_hi
            cache_ptr = self._cache_ptr
            # iterate over complete caches
            while cache_hi < lst_day:
                while cache_ptr < len(cache):
                    yield cache[cache_ptr]
                    cache_ptr += 1
                self._load(date.fromordinal(cache_hi))
                cache = self._cache
                cache_hi = self._cache_hi
                cache_ptr = 0
            # iterate over part of cache
            while cache_ptr < len(cache):
                if cache[cache_ptr]['idx'] >= b:
                    return
                yield cache[cache_ptr]
                cache_ptr += 1
            return
        def __getitem__(self, i):
            """Return the data item or items with index i.

            i must be a datetime object or a slice.
            If i is a single datetime then a value with that index must exist."""
            if isinstance(i, slice):
                return self._get_slice(i)
            if not isinstance(i, datetime):
                raise TypeError("list indices must be %s" % (datetime))
            self._set_cache_ptr(i)
            if (self._cache_ptr >= len(self._cache) or
                self._cache[self._cache_ptr]['idx'] != i):
                raise KeyError(i)
            return self._cache[self._cache_ptr]
        def __setitem__(self, i, x):
            """Store a value x with index i.

            i must be a datetime object.
            If there is already a value with index i, it is overwritten.
            """
            if not isinstance(i, datetime):
                raise TypeError("index '%s' is not %s" % (i, datetime))
            x['idx'] = i
            self._set_cache_ptr(i)
            if len(self._cache) == 0:
                self._fst_day = min(self._fst_day, self._cache_lo)
                self._lst_day = max(self._lst_day, self._cache_hi)
            if (self._cache_ptr < len(self._cache) and
                self._cache[self._cache_ptr]['idx'] == i):
                self._cache[self._cache_ptr] = x
            else:
                self._cache.insert(self._cache_ptr, x)
            self._cache_dirty = True
        def _del_slice(self, i):
            a, b, lst_day = self._slice(i)
            # go to start of slice
            self._set_cache_ptr(a)
            # delete to end of cache
            while self._cache_hi < lst_day:
                del self._cache[self._cache_ptr:]
                self._cache_dirty = True
                self._load(date.fromordinal(self._cache_hi))
                self._cache_ptr = 0
            # delete part of cache
            while (self._cache_ptr < len(self._cache) and
                   self._cache[self._cache_ptr]['idx'] < b):
                del self._cache[self._cache_ptr]
                self._cache_dirty = True
            return
        def __delitem__(self, i):
            """Delete the data item or items with index i.

            i must be a datetime object or a slice.
            If i is a single datetime then a value with that index must exist."""
            if isinstance(i, slice):
                return self._del_slice(i)
            if not isinstance(i, datetime):
                raise TypeError("list indices must be %s" % (datetime))
            self._set_cache_ptr(i)
            if (self._cache_ptr >= len(self._cache) or
                self._cache[self._cache_ptr]['idx'] != i):
                raise KeyError(i)
            del self._cache[self._cache_ptr]
            self._cache_dirty = True
        def before(self, idx):
            """Return datetime of newest existing data record whose
            datetime is < idx.

            Might not even be in the same year! If no such record exists,
            return None."""
            if not isinstance(idx, datetime):
                raise TypeError("'%s' is not %s" % (idx, datetime))
            day = min(idx.toordinal(), self._lst_day - 1)
            while day >= self._fst_day:
                if day < self._cache_lo or day >= self._cache_hi:
                    self._load(date.fromordinal(day))
                self._cache_ptr = self._binary_search(idx, self._cache_ptr)
                if self._cache_ptr > 0:
                    return self._cache[self._cache_ptr - 1]['idx']
                day = self._cache_lo - 1
            return None
        def after(self, idx):
            """Return datetime of oldest existing data record whose
            datetime is >= idx.

            Might not even be in the same year! If no such record exists,
            return None."""
            if not isinstance(idx, datetime):
                raise TypeError("'%s' is not %s" % (idx, datetime))
            day = max(idx.toordinal(), self._fst_day)
            while day < self._lst_day:
                if day < self._cache_lo or day >= self._cache_hi:
                    self._load(date.fromordinal(day))
                self._cache_ptr = self._binary_search(idx, self._cache_ptr)
                if self._cache_ptr < len(self._cache):
                    return self._cache[self._cache_ptr]['idx']
                day = self._cache_hi
            return None
        def nearest(self, idx):
            """Return datetime of record whose datetime is nearest idx."""
            hi = self.after(idx)
            lo = self.before(idx)
            if hi is None:
                return lo
            if lo is None:
                return hi
            if abs(hi - idx) < abs(lo - idx):
                return hi
            return lo
        def _set_cache_ptr(self, i):
            day = i.toordinal()
            if day < self._cache_lo or day >= self._cache_hi:
                self._load(i)
            self._cache_ptr = self._binary_search(i, self._cache_ptr)
        def _binary_search(self, idx, start):
            hi = len(self._cache) - 1
            if hi < 0 or self._cache[0]['idx'] >= idx:
                return 0
            if self._cache[hi]['idx'] < idx:
                return hi + 1
            lo = 0
            start = min(start, hi)
            if self._cache[start]['idx'] < idx:
                lo = start
            else:
                hi = start
            while hi > lo + 1:
                mid = (lo + hi) // 2
                if self._cache[mid]['idx'] < idx:
                    lo = mid
                else:
                    hi = mid
            return hi
        def _load(self, target_date):
            self.flush()
            self._cache = []
            self._cache_ptr = 0
            self._cache_path, self._cache_lo, self._cache_hi = self._get_cache_path(target_date)
            if os.path.exists(self._cache_path):
                if sys.version_info[0] >= 3:
                    csvfile = open(self._cache_path, 'r', newline='')
                else:
                    csvfile = open(self._cache_path, 'rb')
                reader = csv.reader(csvfile, quoting=csv.QUOTE_NONE)
                for row in reader:
                    result = {}
                    for key, value in zip(self.key_list, row):
                        if value == '':
                            result[key] = None
                        else:
                            result[key] = self.conv[key](value)
                    self._cache.append(result)
        def flush(self):
            if not self._cache_dirty:
                return
            self._cache_dirty = False
            if len(self._cache) == 0:
                if os.path.exists(self._cache_path):
                    # existing data has been wiped, so delete file
                    os.unlink(self._cache_path)
                return
            dir = os.path.dirname(self._cache_path)
            if not os.path.isdir(dir):
                os.makedirs(dir)
            if sys.version_info[0] >= 3:
                csvfile = open(self._cache_path, 'w', newline='')
            else:
                csvfile = open(self._cache_path, 'wb')
            writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
            for data in self._cache:
                row = []
                for key in self.key_list[0:len(data)]:
                    row.append(data[key])
                writer.writerow(row)
        def _get_cache_path(self, target_date):
            # default implementation - one file per day
            path = os.path.join(self._root_dir,
                                target_date.strftime("%Y"),
                                target_date.strftime("%Y-%m"),
                                target_date.strftime("%Y-%m-%d.txt"))
            lo = target_date.toordinal()
            hi = lo + 1
            return path, lo, hi
    class data_store(core_store):
        """Stores raw weather station data."""
        def __init__(self, root_dir):
            core_store.__init__(self, os.path.join(root_dir, 'raw'))
        key_list = [
            'idx', 'delay', 'hum_in', 'temp_in', 'hum_out', 'temp_out',
            'abs_pressure', 'wind_ave', 'wind_gust', 'wind_dir', 'rain',
            'status', 'illuminance', 'uv',
            ]
        conv = {
            'idx'          : safestrptime,
            'delay'        : int,
            'hum_in'       : int,
            'temp_in'      : float,
            'hum_out'      : int,
            'temp_out'     : float,
            'abs_pressure' : float,
            'wind_ave'     : float,
            'wind_gust'    : float,
            'wind_dir'     : int,
            'rain'         : float,
            'status'       : int,
            'illuminance'  : float,
            'uv'           : int,
            }
    class calib_store(core_store):
        """Stores 'calibrated' weather station data."""
        def __init__(self, root_dir):
            core_store.__init__(self, os.path.join(root_dir, 'calib'))
        key_list = [
            'idx', 'delay', 'hum_in', 'temp_in', 'hum_out', 'temp_out',
            'abs_pressure', 'rel_pressure', 'wind_ave', 'wind_gust', 'wind_dir',
            'rain', 'status', 'illuminance', 'uv',
            ]
        conv = {
            'idx'          : safestrptime,
            'delay'        : int,
            'hum_in'       : int,
            'temp_in'      : float,
            'hum_out'      : int,
            'temp_out'     : float,
            'abs_pressure' : float,
            'rel_pressure' : float,
            'wind_ave'     : float,
            'wind_gust'    : float,
            'wind_dir'     : int,
            'rain'         : float,
            'status'       : int,
            'illuminance'  : float,
            'uv'           : int,
            }
    class hourly_store(core_store):
        """Stores hourly summary weather station data."""
        def __init__(self, root_dir):
            core_store.__init__(self, os.path.join(root_dir, 'hourly'))
        key_list = [
            'idx', 'hum_in', 'temp_in', 'hum_out', 'temp_out',
            'abs_pressure', 'rel_pressure', 'pressure_trend',
            'wind_ave', 'wind_gust', 'wind_dir', 'rain', 'illuminance', 'uv',
            ]
        conv = {
            'idx'               : safestrptime,
            'hum_in'            : int,
            'temp_in'           : float,
            'hum_out'           : int,
            'temp_out'          : float,
            'abs_pressure'      : float,
            'rel_pressure'      : float,
            'pressure_trend'    : float,
            'wind_ave'          : float,
            'wind_gust'         : float,
            'wind_dir'          : int,
            'rain'              : float,
            'illuminance'       : float,
            'uv'                : int,
            }
    class daily_store(core_store):
        """Stores daily summary weather station data."""
        def __init__(self, root_dir):
            core_store.__init__(self, os.path.join(root_dir, 'daily'))
        key_list = [
            'idx', 'start',
            'hum_out_ave',
            'hum_out_min', 'hum_out_min_t', 'hum_out_max', 'hum_out_max_t',
            'temp_out_ave',
            'temp_out_min', 'temp_out_min_t', 'temp_out_max', 'temp_out_max_t',
            'hum_in_ave',
            'hum_in_min', 'hum_in_min_t', 'hum_in_max', 'hum_in_max_t',
            'temp_in_ave',
            'temp_in_min', 'temp_in_min_t', 'temp_in_max', 'temp_in_max_t',
            'abs_pressure_ave',
            'abs_pressure_min', 'abs_pressure_min_t',
            'abs_pressure_max', 'abs_pressure_max_t',
            'rel_pressure_ave',
            'rel_pressure_min', 'rel_pressure_min_t',
            'rel_pressure_max', 'rel_pressure_max_t',
            'wind_ave', 'wind_gust', 'wind_gust_t', 'wind_dir',
            'rain',
            'illuminance_ave', 'illuminance_max', 'illuminance_max_t',
            'uv_ave', 'uv_max', 'uv_max_t',
            ]
        conv = {
            'idx'                : safestrptime,
            'start'              : safestrptime,
            'hum_out_ave'        : float,
            'hum_out_min'        : int,
            'hum_out_min_t'      : safestrptime,
            'hum_out_max'        : int,
            'hum_out_max_t'      : safestrptime,
            'temp_out_ave'       : float,
            'temp_out_min'       : float,
            'temp_out_min_t'     : safestrptime,
            'temp_out_max'       : float,
            'temp_out_max_t'     : safestrptime,
            'hum_in_ave'         : float,
            'hum_in_min'         : int,
            'hum_in_min_t'       : safestrptime,
            'hum_in_max'         : int,
            'hum_in_max_t'       : safestrptime,
            'temp_in_ave'        : float,
            'temp_in_min'        : float,
            'temp_in_min_t'      : safestrptime,
            'temp_in_max'        : float,
            'temp_in_max_t'      : safestrptime,
            'abs_pressure_ave'   : float,
            'abs_pressure_min'   : float,
            'abs_pressure_min_t' : safestrptime,
            'abs_pressure_max'   : float,
            'abs_pressure_max_t' : safestrptime,
            'rel_pressure_ave'   : float,
            'rel_pressure_min'   : float,
            'rel_pressure_min_t' : safestrptime,
            'rel_pressure_max'   : float,
            'rel_pressure_max_t' : safestrptime,
            'wind_ave'           : float,
            'wind_gust'          : float,
            'wind_gust_t'        : safestrptime,
            'wind_dir'           : int,
            'rain'               : float,
            'illuminance_ave'    : float,
            'illuminance_max'    : float,
            'illuminance_max_t'  : safestrptime,
            'uv_ave'             : float,
            'uv_max'             : int,
            'uv_max_t'           : safestrptime,
            }
        def _get_cache_path(self, target_date):
            # one file per month
            path = os.path.join(self._root_dir,
                                target_date.strftime("%Y"),
                                target_date.strftime("%Y-%m-01.txt"))
            lo = target_date.replace(day=1)
            if lo.month < 12:
                hi = lo.replace(month=lo.month+1)
            else:
                hi = lo.replace(year=lo.year+1, month=1)
            return path, lo.toordinal(), hi.toordinal()
    class monthly_store(core_store):
        """Stores monthly summary weather station data."""
        def __init__(self, root_dir):
            core_store.__init__(self, os.path.join(root_dir, 'monthly'))
        key_list = [
            'idx', 'start',
            'hum_out_ave',
            'hum_out_min', 'hum_out_min_t', 'hum_out_max', 'hum_out_max_t',
            'temp_out_ave',
            'temp_out_min_lo', 'temp_out_min_lo_t',
            'temp_out_min_hi', 'temp_out_min_hi_t', 'temp_out_min_ave',
            'temp_out_max_lo', 'temp_out_max_lo_t',
            'temp_out_max_hi', 'temp_out_max_hi_t', 'temp_out_max_ave',
            'hum_in_ave',
            'hum_in_min', 'hum_in_min_t', 'hum_in_max', 'hum_in_max_t',
            'temp_in_ave',
            'temp_in_min_lo', 'temp_in_min_lo_t',
            'temp_in_min_hi', 'temp_in_min_hi_t', 'temp_in_min_ave',
            'temp_in_max_lo', 'temp_in_max_lo_t',
            'temp_in_max_hi', 'temp_in_max_hi_t', 'temp_in_max_ave',
            'abs_pressure_ave',
            'abs_pressure_min', 'abs_pressure_min_t',
            'abs_pressure_max', 'abs_pressure_max_t',
            'rel_pressure_ave',
            'rel_pressure_min', 'rel_pressure_min_t',
            'rel_pressure_max', 'rel_pressure_max_t',
            'wind_ave', 'wind_gust', 'wind_gust_t', 'wind_dir',
            'rain', 'rain_days',
            'illuminance_ave',
            'illuminance_max_lo', 'illuminance_max_lo_t',
            'illuminance_max_hi', 'illuminance_max_hi_t', 'illuminance_max_ave',
            'uv_ave',
            'uv_max_lo', 'uv_max_lo_t', 'uv_max_hi', 'uv_max_hi_t', 'uv_max_ave',
            ]
        conv = {
            'idx'                  : safestrptime,
            'start'                : safestrptime,
            'hum_out_ave'          : float,
            'hum_out_min'          : int,
            'hum_out_min_t'        : safestrptime,
            'hum_out_max'          : int,
            'hum_out_max_t'        : safestrptime,
            'temp_out_ave'         : float,
            'temp_out_min_lo'      : float,
            'temp_out_min_lo_t'    : safestrptime,
            'temp_out_min_hi'      : float,
            'temp_out_min_hi_t'    : safestrptime,
            'temp_out_min_ave'     : float,
            'temp_out_max_lo'      : float,
            'temp_out_max_lo_t'    : safestrptime,
            'temp_out_max_hi'      : float,
            'temp_out_max_hi_t'    : safestrptime,
            'temp_out_max_ave'     : float,
            'hum_in_ave'           : float,
            'hum_in_min'           : int,
            'hum_in_min_t'         : safestrptime,
            'hum_in_max'           : int,
            'hum_in_max_t'         : safestrptime,
            'temp_in_ave'          : float,
            'temp_in_min_lo'       : float,
            'temp_in_min_lo_t'     : safestrptime,
            'temp_in_min_hi'       : float,
            'temp_in_min_hi_t'     : safestrptime,
            'temp_in_min_ave'      : float,
            'temp_in_max_lo'       : float,
            'temp_in_max_lo_t'     : safestrptime,
            'temp_in_max_hi'       : float,
            'temp_in_max_hi_t'     : safestrptime,
            'temp_in_max_ave'      : float,
            'abs_pressure_ave'     : float,
            'abs_pressure_min'     : float,
            'abs_pressure_min_t'   : safestrptime,
            'abs_pressure_max'     : float,
            'abs_pressure_max_t'   : safestrptime,
            'rel_pressure_ave'     : float,
            'rel_pressure_min'     : float,
            'rel_pressure_min_t'   : safestrptime,
            'rel_pressure_max'     : float,
            'rel_pressure_max_t'   : safestrptime,
            'wind_ave'             : float,
            'wind_gust'            : float,
            'wind_gust_t'          : safestrptime,
            'wind_dir'             : int,
            'rain'                 : float,
            'rain_days'            : int,
            'illuminance_ave'      : float,
            'illuminance_max_lo'   : float,
            'illuminance_max_lo_t' : safestrptime,
            'illuminance_max_hi'   : float,
            'illuminance_max_hi_t' : safestrptime,
            'illuminance_max_ave'  : float,
            'uv_ave'               : float,
            'uv_max_lo'            : int,
            'uv_max_lo_t'          : safestrptime,
            'uv_max_hi'            : int,
            'uv_max_hi_t'          : safestrptime,
            'uv_max_ave'           : float,
            }
        def _get_cache_path(self, target_date):
            # one file per year
            path = os.path.join(self._root_dir,
                                target_date.strftime("%Y-01-01.txt"))
            lo = target_date.replace(month=1, day=1)
            hi = lo.replace(year=lo.year+1)
            return path, lo.toordinal(), hi.toordinal()

### Navigation

- index
- modules |
- pywws 13.06_r1023 documentation »
- Code du module »