from collections import defaultdict
from contextlib import contextmanager
from datetime import timedelta
import io
import os
import stat
from unittest.mock import MagicMock

import pytest

import confluent_kafka
from hop import models
from hop.io import StartPosition


# example GCN circular from https://gcn.gsfc.nasa.gov/gcn3_circulars.html
GCN_TITLE = "GCN GRB OBSERVATION REPORT"
GCN_NUMBER = "40"
GCN_SUBJECT = "GRB980329 VLA observations"
GCN_DATE = "98/04/03 07:10:15 GMT"
GCN_FROM = "Greg Taylor at NRAO"
GCN_BODY = """
G.B. Taylor, D.A. Frail (NRAO), S.R. Kulkarni (Caltech), and
the BeppoSAX GRB team report:

We have observed the field containing the proposed x-ray counterpart
1SAX J0702.6+3850 of GRB 980329 (IAUC 6854) with the VLA at 8.4 GHz
on UT 1998 Mar 30.2, April 1.1, and April 2.1.  Observations on April
1.1 detected a radio source VLA J0702+3850 within the 1 arcminute
error circle of 1SAX J0702.6+3850.  The coordinates of
VLA J0702+3850 are: ra = 07h02m38.02170s dec = 38d50'44.0170" (equinox
J2000) with an uncertainty of 0.05 arcsec in each coordinate.  The
size of this radio source is less than 0.25 arcsec.  The density of
sources on the sky stronger than 250 microJy at this frequency is
0.0145 arcmin**-2.

The flux density measurements of VLA J0702+3850 are as follows:

Date(UT)   8.4 GHz Flux Density
--------   ----------------------
Mar 30.2   166 +/- 50 microJy
Apr  1.1   248 +/- 16    "
Apr  2.1    65 +/- 25    "

where the uncertainty in the measurement reflects the 1 sigma rms
noise in the image.  These measurements clearly demonstrate that
the radio source is variable on timescales of less than 1 day.
This rapid variability is similar to that observed in the
radio afterglow from GRB 970508.  We propose VLA J0702+3850
is the radio afterglow from GRB 980329.

Additional radio observations are in progress.
"""

GCN_CIRCULAR = f"""\
TITLE:   {GCN_TITLE}
NUMBER:  {GCN_NUMBER}
SUBJECT: {GCN_SUBJECT}
DATE:    {GCN_DATE}
FROM:    {GCN_FROM}

{GCN_BODY}\
"""

VOEVENT_XML = """\
<?xml version='1.0' encoding='UTF-8'?>
<voe:VOEvent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:voe="http://www.ivoa.net/xml/VOEvent/v2.0" xsi:schemaLocation="http://www.ivoa.net/xml/VOEvent/v2.0 http://www.ivoa.net/xml/VOEvent/VOEvent-v2.0.xsd" version="2.0" role="observation" ivorn="ivo://gwnet/LVC#S200302c-1-Preliminary">
  <Who>
    <Date>2020-03-02T02:00:09</Date>
    <Author>
      <contactName>LIGO Scientific Collaboration and Virgo Collaboration</contactName>
    </Author>
  </Who>
  <What>
    <Param dataType="int" name="Packet_Type" value="150">
      <Description>The Notice Type number is assigned/used within GCN, eg type=150 is an LVC_PRELIMINARY notice</Description>
    </Param>
    <Param dataType="int" name="internal" value="0">
      <Description>Indicates whether this event should be distributed to LSC/Virgo members only</Description>
    </Param>
    <Param dataType="int" name="Pkt_Ser_Num" value="1">
      <Description>A number that increments by 1 each time a new revision is issued for this event</Description>
    </Param>
    <Param dataType="string" name="GraceID" ucd="meta.id" value="S200302c">
      <Description>Identifier in GraceDB</Description>
    </Param>
    <Param dataType="string" name="AlertType" ucd="meta.version" value="Preliminary">
      <Description>VOEvent alert type</Description>
    </Param>
    <Param dataType="int" name="HardwareInj" ucd="meta.number" value="0">
      <Description>Indicates that this event is a hardware injection if 1, no if 0</Description>
    </Param>
    <Param dataType="int" name="OpenAlert" ucd="meta.number" value="1">
      <Description>Indicates that this event is an open alert if 1, no if 0</Description>
    </Param>
    <Param dataType="string" name="EventPage" ucd="meta.ref.url" value="https://gracedb.ligo.org/superevents/S200302c/view/">
      <Description>Web page for evolving status of this GW candidate</Description>
    </Param>
    <Param dataType="string" name="Instruments" ucd="meta.code" value="H1,V1">
      <Description>List of instruments used in analysis to identify this event</Description>
    </Param>
    <Param dataType="float" name="FAR" ucd="arith.rate;stat.falsealarm" unit="Hz" value="9.349090689402942e-09">
      <Description>False alarm rate for GW candidates with this strength or greater</Description>
    </Param>
    <Param dataType="string" name="Group" ucd="meta.code" value="CBC">
      <Description>Data analysis working group</Description>
    </Param>
    <Param dataType="string" name="Pipeline" ucd="meta.code" value="gstlal">
      <Description>Low-latency data analysis pipeline</Description>
    </Param>
    <Param dataType="string" name="Search" ucd="meta.code" value="AllSky">
      <Description>Specific low-latency search</Description>
    </Param>
    <Group name="GW_SKYMAP" type="GW_SKYMAP">
      <Param dataType="string" name="skymap_fits" ucd="meta.ref.url" value="https://gracedb.ligo.org/api/superevents/S200302c/files/bayestar.fits.gz,0">
        <Description>Sky Map FITS</Description>
      </Param>
    </Group>
    <Group name="Classification" type="Classification">
      <Param dataType="float" name="BNS" ucd="stat.probability" value="0.0">
        <Description>Probability that the source is a binary neutron star merger (both objects lighter than 3 solar masses)</Description>
      </Param>
      <Param dataType="float" name="NSBH" ucd="stat.probability" value="0.0">
        <Description>Probability that the source is a neutron star-black hole merger (primary heavier than 5 solar masses, secondary lighter than 3 solar masses)</Description>
      </Param>
      <Param dataType="float" name="BBH" ucd="stat.probability" value="0.8895532192171397">
        <Description>Probability that the source is a binary black hole merger (both objects heavier than 5 solar masses)</Description>
      </Param>
      <Param dataType="float" name="MassGap" ucd="stat.probability" value="0.0">
        <Description>Probability that the source has at least one object between 3 and 5 solar masses</Description>
      </Param>
      <Param dataType="float" name="Terrestrial" ucd="stat.probability" value="0.11044678078286028">
        <Description>Probability that the source is terrestrial (i.e., a background noise fluctuation or a glitch)</Description>
      </Param>
      <Description>Source classification: binary neutron star (BNS), neutron star-black hole (NSBH), binary black hole (BBH), MassGap, or terrestrial (noise)</Description>
    </Group>
    <Group name="Properties" type="Properties">
      <Param dataType="float" name="HasNS" ucd="stat.probability" value="0.0">
        <Description>Probability that at least one object in the binary has a mass that is less than 3 solar masses</Description>
      </Param>
      <Param dataType="float" name="HasRemnant" ucd="stat.probability" value="0.0">
        <Description>Probability that a nonzero mass was ejected outside the central remnant object</Description>
      </Param>
      <Description>Qualitative properties of the source, conditioned on the assumption that the signal is an astrophysical compact binary merger</Description>
    </Group>
  </What>
  <WhereWhen>
    <ObsDataLocation>
      <ObservatoryLocation id="LIGO Virgo"/>
      <ObservationLocation>
        <AstroCoordSystem id="UTC-FK5-GEO"/>
        <AstroCoords coord_system_id="UTC-FK5-GEO">
          <Time unit="s">
            <TimeInstant>
              <ISOTime>2020-03-02T01:58:11.519119</ISOTime>
            </TimeInstant>
          </Time>
        </AstroCoords>
      </ObservationLocation>
    </ObsDataLocation>
  </WhereWhen>
  <Description>Report of a candidate gravitational wave event</Description>
  <How>
    <Description>Candidate gravitational wave event identified by low-latency analysis</Description>
    <Description>V1: Virgo 3 km gravitational wave detector</Description>
    <Description>H1: LIGO Hanford 4 km gravitational wave detector</Description>
  </How>
</voe:VOEvent>\
"""

VOEVENT_JSON = """\
{"ivorn": "ivo://nasa.gsfc.gcn/SWIFT#Actual_Point_Dir_2025-07-30T17:17:19.67_1442872420-151", "role": "utility", "version": "2.0", "Who": {"AuthorIVORN": "ivo://nasa.gsfc.tan/gcn", "Author": {"shortName": "VO-GCN", "contactName": "Scott Barthelmy", "contactPhone": "+1-301-286-3106", "contactEmail": "scott.barthelmy@nasa.gov"}, "Date": "2025-07-30T17:17:14", "Description": "This VOEvent message was created with GCN VOE version: 15.08 17jun22"}, "What": {"Param": [{"name": "Packet_Type", "value": "103"}, {"name": "Pkt_Ser_Num", "value": "303"}, {"name": "TrigID", "value": "31844", "ucd": "meta.id"}, {"name": "Segment_Num", "value": "86", "ucd": "meta.id.part"}, {"name": "Slew_TJD", "value": "20886", "unit": "days", "ucd": "time"}, {"name": "Slew_SOD", "value": "62239.67", "unit": "sec", "ucd": "time"}, {"name": "Slew_RA", "value": "332.1521", "unit": "deg", "ucd": "pos.eq.ra"}, {"name": "Slew_Dec", "value": "-47.1956", "unit": "deg", "ucd": "pos.eq.dec"}, {"name": "Slew_Roll", "value": "126.2206", "unit": "deg"}, {"name": "SC_Long", "value": "74.15", "unit": "deg", "ucd": "pos.earth.lon"}, {"name": "SC_Lat", "value": "-4.83", "unit": "deg", "ucd": "pos.earth.lat"}, {"name": "Misc_flags", "value": "0x0"}, {"name": "Coords_Type", "value": "2", "unit": "dn"}, {"name": "Coords_String", "value": "pointing_direction"}], "Group": [{"name": "Misc_Flags", "Param": [{"name": "Near_Bright_Star", "value": "false"}, {"name": "Flt_Generated", "value": "true"}, {"name": "Gnd_Generated", "value": "false"}]}, {"name": "Obs_Support_Info", "Description": "The Sun and Moon values are valid at the time the VOEvent XML message was created.", "Param": [{"name": "Sun_RA", "value": "130.21", "unit": "deg", "ucd": "pos.eq.ra"}, {"name": "Sun_Dec", "value": "18.32", "unit": "deg", "ucd": "pos.eq.dec"}, {"name": "Sun_Distance", "value": "145.91", "unit": "deg", "ucd": "pos.angDistance"}, {"name": "Sun_Hr_Angle", "value": "10.51", "unit": "hr"}, {"name": "Moon_RA", "value": "195.69", "unit": "deg", "ucd": "pos.eq.ra"}, {"name": "Moon_Dec", "value": "-9.46", "unit": "deg", "ucd": "pos.eq.dec"}, {"name": "MOON_Distance", "value": "111.71", "unit": "deg", "ucd": "pos.angDistance"}, {"name": "Moon_Illum", "value": "33.28", "unit": "%", "ucd": "arith.ratio"}, {"name": "Galactic_Long", "value": "349.60", "unit": "deg", "ucd": "pos.galactic.lon"}, {"name": "Galactic_Lat", "value": "-52.46", "unit": "deg", "ucd": "pos.galactic.lat"}, {"name": "Ecliptic_Long", "value": "315.86", "unit": "deg", "ucd": "pos.ecliptic.lon"}, {"name": "Ecliptic_Lat", "value": "-33.15", "unit": "deg", "ucd": "pos.ecliptic.lat"}]}], "Description": "This is the current pointing direction for the Swift spacecraft after settling from an actual slew."}, "WhereWhen": {"ObsDataLocation": {"ObservatoryLocation": {"id": "GEOLUN"}, "ObservationLocation": {"AstroCoordSystem": {"id": "UTC-FK5-GEO"}, "AstroCoords": {"coord_system_id": "UTC-FK5-GEO", "Time": {"unit": "s", "TimeInstant": {"ISOTime": "2025-07-30T17:17:19.67"}}, "Position2D": {"unit": "deg", "Name1": "RA", "Name2": "Dec", "Value2": {"C1": "332.1521", "C2": "-47.1956"}, "Error2Radius": "0.0000"}}}}, "Description": "The RA,Dec coordinates are of the type: pointing_direction."}, "How": {"Description": "Swift Satellite", "Reference": {"uri": "http://gcn.gsfc.nasa.gov/swift.html", "type": "url"}}, "Why": {"Inference": {"probability": "1.00", "Concept": "Swift spacecraft just slewed to an observing target."}}, "Citations": {}, "Description": null, "Reference": {}}\
"""

MESSAGE_GCN_TEXT = \
    b'TITLE:            GCN/AMON NOTICE\n' \
    b'NOTICE_DATE:      Fri 12 Apr 24 05:34:35 UT\n' \
    b'NOTICE_TYPE:      ICECUBE Astrotrack Bronze \n' \
    b'STREAM:           25\n' \
    b'RUN_NUM:          139279\n' \
    b'EVENT_NUM:        10803235\n' \
    b'SRC_RA:           103.7861d {+06h 55m 09s} (J2000),\n' \
    b'                  104.1106d {+06h 56m 27s} (current),\n' \
    b'                  103.1176d {+06h 52m 28s} (1950)\n' \
    b'SRC_DEC:           +5.8716d {+05d 52\' 18"} (J2000),\n' \
    b'                   +5.8390d {+05d 50\' 20"} (current),\n' \
    b'                   +5.9364d {+05d 56\' 11"} (1950)\n' \
    b'SRC_ERROR:        69.22 [arcmin radius, stat-only, 90% containment]\n' \
    b'SRC_ERROR50:      26.96 [arcmin radius, stat-only, 50% containment]\n' \
    b'DISCOVERY_DATE:   20412 TJD;   103 DOY;   24/04/12 (yy/mm/dd)\n' \
    b'DISCOVERY_TIME:   20026 SOD {05:33:46.89} UT\n' \
    b'REVISION:         0\n' \
    b'ENERGY:           1.2126e+02 [TeV]\n' \
    b'SIGNALNESS:       3.0910e-01 [dn]\n' \
    b'FAR:              3.4096 [yr^-1]\n' \
    b'SUN_POSTN:         21.10d {+01h 24m 24s}   +8.87d {+08d 52\' 11"}\n' \
    b'SUN_DIST:          82.22 [deg]   Sun_angle= -5.5 [hr] (East of Sun)\n' \
    b'MOON_POSTN:        67.46d {+04h 29m 50s}  +26.13d {+26d 07\' 33"}\n' \
    b'MOON_DIST:         40.41 [deg]\n' \
    b'GAL_COORDS:       208.12,  3.50 [deg] galactic lon,lat of the event\n' \
    b'ECL_COORDS:       104.34,-16.88 [deg] ecliptic lon,lat of the event\n' \
    b'COMMENTS:         IceCube Bronze event.  \n' \
    b'COMMENTS:         The position error is statistical only, there is no systematic added.  '


MESSAGE_BLOB = b"This is a sample blob message. It is unstructured and does not require special parsing."

MESSAGE_JSON = b'{"foo":"bar", "baz":5}'

MESSAGE_AVRO = \
    b'\x4f\x62\x6a\x01\x04\x14\x61\x76\x72\x6f\x2e\x63' \
    b'\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x16\x61\x76' \
    b'\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x8c\x0b\x7b' \
    b'\x22\x74\x79\x70\x65\x22\x3a\x20\x22\x72\x65\x63' \
    b'\x6f\x72\x64\x22\x2c\x20\x22\x6e\x61\x6d\x65\x22' \
    b'\x3a\x20\x22\x61\x75\x74\x6f\x5f\x72\x65\x63\x6f' \
    b'\x72\x64\x32\x22\x2c\x20\x22\x66\x69\x65\x6c\x64' \
    b'\x73\x22\x3a\x20\x5b\x7b\x22\x74\x79\x70\x65\x22' \
    b'\x3a\x20\x7b\x22\x74\x79\x70\x65\x22\x3a\x20\x22' \
    b'\x61\x72\x72\x61\x79\x22\x2c\x20\x22\x69\x74\x65' \
    b'\x6d\x73\x22\x3a\x20\x22\x6c\x6f\x6e\x67\x22\x7d' \
    b'\x2c\x20\x22\x6e\x61\x6d\x65\x22\x3a\x20\x22\x61' \
    b'\x72\x72\x22\x7d\x2c\x20\x7b\x22\x74\x79\x70\x65' \
    b'\x22\x3a\x20\x7b\x22\x74\x79\x70\x65\x22\x3a\x20' \
    b'\x22\x61\x72\x72\x61\x79\x22\x2c\x20\x22\x69\x74' \
    b'\x65\x6d\x73\x22\x3a\x20\x7b\x22\x74\x79\x70\x65' \
    b'\x22\x3a\x20\x22\x6d\x61\x70\x22\x2c\x20\x22\x76' \
    b'\x61\x6c\x75\x65\x73\x22\x3a\x20\x22\x6c\x6f\x6e' \
    b'\x67\x22\x7d\x7d\x2c\x20\x22\x6e\x61\x6d\x65\x22' \
    b'\x3a\x20\x22\x73\x75\x62\x5f\x6f\x62\x6a\x65\x63' \
    b'\x74\x73\x22\x7d\x2c\x20\x7b\x22\x74\x79\x70\x65' \
    b'\x22\x3a\x20\x7b\x22\x74\x79\x70\x65\x22\x3a\x20' \
    b'\x22\x72\x65\x63\x6f\x72\x64\x22\x2c\x20\x22\x6e' \
    b'\x61\x6d\x65\x22\x3a\x20\x22\x61\x75\x74\x6f\x5f' \
    b'\x72\x65\x63\x6f\x72\x64\x31\x22\x2c\x20\x22\x66' \
    b'\x69\x65\x6c\x64\x73\x22\x3a\x20\x5b\x7b\x22\x74' \
    b'\x79\x70\x65\x22\x3a\x20\x22\x73\x74\x72\x69\x6e' \
    b'\x67\x22\x2c\x20\x22\x6e\x61\x6d\x65\x22\x3a\x20' \
    b'\x22\x66\x6f\x6f\x22\x7d\x2c\x20\x7b\x22\x74\x79' \
    b'\x70\x65\x22\x3a\x20\x22\x6c\x6f\x6e\x67\x22\x2c' \
    b'\x20\x22\x6e\x61\x6d\x65\x22\x3a\x20\x22\x62\x61' \
    b'\x72\x22\x7d\x2c\x20\x7b\x22\x74\x79\x70\x65\x22' \
    b'\x3a\x20\x22\x6e\x75\x6c\x6c\x22\x2c\x20\x22\x6e' \
    b'\x61\x6d\x65\x22\x3a\x20\x22\x62\x61\x7a\x22\x7d' \
    b'\x2c\x20\x7b\x22\x74\x79\x70\x65\x22\x3a\x20\x7b' \
    b'\x22\x74\x79\x70\x65\x22\x3a\x20\x22\x72\x65\x63' \
    b'\x6f\x72\x64\x22\x2c\x20\x22\x6e\x61\x6d\x65\x22' \
    b'\x3a\x20\x22\x61\x75\x74\x6f\x5f\x72\x65\x63\x6f' \
    b'\x72\x64\x30\x22\x2c\x20\x22\x66\x69\x65\x6c\x64' \
    b'\x73\x22\x3a\x20\x5b\x7b\x22\x74\x79\x70\x65\x22' \
    b'\x3a\x20\x22\x62\x79\x74\x65\x73\x22\x2c\x20\x22' \
    b'\x6e\x61\x6d\x65\x22\x3a\x20\x22\x78\x65\x6e\x22' \
    b'\x7d\x2c\x20\x7b\x22\x74\x79\x70\x65\x22\x3a\x20' \
    b'\x7b\x22\x74\x79\x70\x65\x22\x3a\x20\x22\x61\x72' \
    b'\x72\x61\x79\x22\x2c\x20\x22\x69\x74\x65\x6d\x73' \
    b'\x22\x3a\x20\x22\x6c\x6f\x6e\x67\x22\x7d\x2c\x20' \
    b'\x22\x6e\x61\x6d\x65\x22\x3a\x20\x22\x68\x6f\x6d' \
    b'\x22\x7d\x2c\x20\x7b\x22\x74\x79\x70\x65\x22\x3a' \
    b'\x20\x22\x64\x6f\x75\x62\x6c\x65\x22\x2c\x20\x22' \
    b'\x6e\x61\x6d\x65\x22\x3a\x20\x22\x64\x72\x65\x6c' \
    b'\x22\x7d\x5d\x7d\x2c\x20\x22\x6e\x61\x6d\x65\x22' \
    b'\x3a\x20\x22\x71\x75\x75\x78\x22\x7d\x5d\x7d\x2c' \
    b'\x20\x22\x6e\x61\x6d\x65\x22\x3a\x20\x22\x74\x68' \
    b'\x69\x6e\x67\x79\x22\x7d\x2c\x20\x7b\x22\x74\x79' \
    b'\x70\x65\x22\x3a\x20\x22\x62\x79\x74\x65\x73\x22' \
    b'\x2c\x20\x22\x6e\x61\x6d\x65\x22\x3a\x20\x22\x64' \
    b'\x61\x74\x61\x22\x7d\x2c\x20\x7b\x22\x74\x79\x70' \
    b'\x65\x22\x3a\x20\x7b\x22\x74\x79\x70\x65\x22\x3a' \
    b'\x20\x22\x61\x72\x72\x61\x79\x22\x2c\x20\x22\x69' \
    b'\x74\x65\x6d\x73\x22\x3a\x20\x22\x62\x6f\x6f\x6c' \
    b'\x65\x61\x6e\x22\x7d\x2c\x20\x22\x6e\x61\x6d\x65' \
    b'\x22\x3a\x20\x22\x6c\x6f\x67\x69\x63\x22\x7d\x5d' \
    b'\x7d\x00\xfd\x2b\x62\xfc\xdf\xb1\xd2\x03\x2e\x33' \
    b'\x42\xea\xa7\x2a\xc4\x52\x02\x70\x08\x02\x04\x06' \
    b'\x08\x00\x04\x04\x02\x61\x02\x02\x62\x04\x00\x04' \
    b'\x02\x63\x06\x02\x64\x08\x00\x00\x06\x61\x62\x63' \
    b'\x2c\x06\x64\x65\x66\x06\xb2\x01\x5c\x0a\x00\x58' \
    b'\x39\xb4\xc8\x76\xbe\x05\x40\x08\x41\x00\x42\x04' \
    b'\x04\x01\x00\x00\xfd\x2b\x62\xfc\xdf\xb1\xd2\x03' \
    b'\x2e\x33\x42\xea\xa7\x2a\xc4\x52'

# Equivalent to the data encoded in MESSAGE_AVRO
AVRO_DATA_EQUIVALENT = {
    "arr": [1, 2, 3, 4],
    "sub_objects": [{"a": 1, "b": 2}, {"c": 3, "d": 4}],
    "thingy": {
        "foo": "abc",
        "bar": 22,
        "baz": None,
        "quux": {"xen": b"def", "hom": [89, 46, 5], "drel": 2.718}
    },
    "data": b"A\x00B\x04",
    "logic": [True, False],
}

GENERAL_CONFIG = """\
[config]
fetch_external = false
automatic_offload = false
"""

# This was the original configuration structure, which permitted only a single credential
AUTH_CONFIG_LEGACY = """\
[auth]
username = "username"
password = "password"
"""

# This is the new configuration structure, which contains a list of credentials
AUTH_CONFIG = """
auth = [{
         username="username",
         password="password"
         }]
"""

# This is the OIDC configuration structure, which is very similar
AUTH_CONFIG_OIDC = """
auth = [{
         username="username",
         password="password",
         token_endpoint="https://example.com/oauth2/token"
         }]
"""


class MockBroker:
    """Mock a Kafka broker.

    This stores internally messages and tracks offsets
    for different consumer groups.

    """

    def __init__(self, default_max_size=1024 * 1024):
        self._default_max_size = default_max_size
        self.reset()

    def reset(self):
        self._messages = defaultdict(list)
        self._offsets = defaultdict(dict)
        self._max_sizes = defaultdict(lambda: self._default_max_size)

    def set_default_topic_max_message_size(self, size):
        self._default_max_size = size
        new_sizes = defaultdict(lambda: self._default_max_size)
        for topic in self._max_sizes.keys():
            new_sizes[topic] = self._max_sizes[topic]
        self._max_sizes = new_sizes

    def set_topic_max_message_size(self, topic, size):
        self._max_sizes[topic] = size

    def get_topic_max_message_size(self, topic):
        return self._max_sizes[topic]

    def insert_message(self, topic, msg, headers=[], key=None):
        self._messages[topic].append((msg, headers, key))

    def write(self, topic, msg, headers=[], delivery_callback=None, key=None):
        assert delivery_callback is not None
        if len(msg) > self._max_sizes[topic]:
            if delivery_callback is None:
                raise RuntimeError("Write failed but no delivery callback set to handle the error")
            err = confluent_kafka.KafkaError(confluent_kafka.KafkaError.MSG_SIZE_TOO_LARGE,
                                             "Message too large", False, False, True)
            delivery_callback(err, None)  # TODO: synthetic message object?
            return
        self._messages[topic].append((msg, headers, key))

    def has_message(self, topic, message, headers=[], key=None):
        return (message, headers, key) in self._messages[topic]

    def read(self, topics, groupid, start_at=StartPosition.EARLIEST, **kwargs):
        if isinstance(topics, str):
            topics = [topics]

        for topic in topics:
            if topic not in self._offsets or groupid not in self._offsets[topic]:
                if start_at == StartPosition.EARLIEST:
                    self._offsets[topic][groupid] = 0
                else:
                    self._offsets[topic][groupid] = len(self._messages[topic]) - 1

            try:
                offset = self._offsets[topic][groupid]
                self._offsets[topic][groupid] += 1
                yield from self._messages[topic][offset:]
            except IndexError:
                pass


@pytest.fixture()
def mock_broker(**kwargs):
    return MockBroker(**kwargs)


class ProducerBrokerWrapper:
    def __init__(self, broker, topic):
        self.broker = broker
        self.topic = topic
        self._delay_sending = 0.0
        self._delayed = []

    def delay_sending(self, delay=1.0):
        """Simulate messages taking time to send"""
        self._delay_sending = delay

    def write(self, msg, headers=[], delivery_callback=None, topic=None, key=None):
        if topic is None:
            if self.topic is not None:
                topic = self.topic
            else:
                raise Exception("No topic specified for write")
        if self._delay_sending > 0.0:
            self._delayed.append((msg, headers, delivery_callback, topic, key))
            return
        self.broker.write(topic, msg, headers, delivery_callback=delivery_callback, key=key)

    def send_delayed(self):
        for msg, headers, delivery_callback, topic, key in self._delayed:
            self.broker.write(topic, msg, headers, delivery_callback=delivery_callback,
                              key=key)
        self._delayed.clear()

    def __len__(self):
        return len(self._delayed)

    def queued_message_count(self):
        return len(self._delayed)

    def flush(self, timeout=None):
        """Simulate time passing, possibly allowing delayed messages to send"""
        if self._delay_sending > 0.0:
            if timeout is None:
                self._delay_sending = 0.0
            else:
                self._delay_sending -= timeout.total_seconds()
                if self._delay_sending < 0.0:
                    self._delay_sending = 0.0
            if self._delay_sending == 0.0:
                self.send_delayed()
        return len(self._delayed)

    def close(self, timeout: timedelta = timedelta(seconds=0)):
        return self.flush(timeout)

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        pass


@pytest.fixture()
def mock_producer():
    def _mock_producer(mock_broker, topic):
        producer = ProducerBrokerWrapper(mock_broker, topic)
        return producer

    return _mock_producer


class MockMessage:
    def __init__(self, value, headers=[]):
        self._value = value
        self._headers = headers

    def value(self):
        return self._value

    def headers(self):
        return self._headers


@pytest.fixture()
def mock_consumer():
    def _mock_consumer(mock_broker, topics, group_id, start_at=StartPosition.EARLIEST):
        class ConsumerBrokerWrapper:
            def __init__(self, broker, topics, group_id, start_at):
                if isinstance(topics, str):
                    topics = [topics]
                self.broker = broker
                self.topics = topics
                self.group_id = group_id
                self.start_at = start_at

            def subscribe(self, topics):
                if isinstance(topics, str):
                    topics = [topics]
                for topic in topics:
                    if topic not in self.topics:
                        self.topics.append(topic)

            def stream(self, *args, **kwargs):
                for message in self.broker.read(self.topics, self.group_id, self.start_at, **kwargs):
                    yield MockMessage(message[0], message[1])

            def close(self):
                pass

            def __enter__(self):
                return self

            def __exit__(self, *exc):
                pass

        consumer = ConsumerBrokerWrapper(mock_broker, topics, group_id, start_at)
        return consumer

    return _mock_consumer


@pytest.fixture()
def mock_admin_client():
    from confluent_kafka.admin import ConfigResource, ConfigEntry, ResourceType

    def _mock_admin_client(mock_broker):
        class MockAdminClient:
            def __init__(self, broker):
                self.broker = broker
                self.describe_configs_queries = 0

            def describe_configs(self, queries):
                futures = {}
                for query in queries:
                    if query.restype != ResourceType.TOPIC:
                        raise ValueError("Only TOPIC queries are emulated")
                    f = MagicMock()
                    max_message_bytes = self.broker.get_topic_max_message_size(query.name)
                    f.result = MagicMock(return_value={"max.message.bytes": ConfigEntry(
                        "max.message.bytes", max_message_bytes)})
                    futures[ConfigResource(query.restype, query.name)] = f
                    self.describe_configs_queries += 1
                return futures

            def reset_query_counter(self):
                self.describe_configs_queries = 0

        return MockAdminClient(mock_broker)

    return _mock_admin_client


@pytest.fixture(scope="session")
def mock_kafka_message():
    message = MagicMock()
    message.topic.return_value = "test-topic"
    message.partition.return_value = 0
    message.offset.return_value = 0
    message.timestamp.return_value = (0, 1234567890)
    message.key.return_value = "test-key"
    message.headers.return_value = [("a header", "a value"), ("another header", "other value")]
    return message


@pytest.fixture(scope="session")
def general_config():
    return GENERAL_CONFIG


@pytest.fixture(scope="session")
def legacy_auth_config():
    return AUTH_CONFIG_LEGACY


@pytest.fixture(scope="session")
def auth_config():
    return AUTH_CONFIG


@pytest.fixture(scope="session")
def auth_config_oidc():
    return AUTH_CONFIG_OIDC


@pytest.fixture(scope="session")
def circular_data_raw():
    return {
        "header": {
            "title": GCN_TITLE,
            "number": GCN_NUMBER,
            "subject": GCN_SUBJECT,
            "date": GCN_DATE,
            "from": GCN_FROM,
        },
        "body": GCN_BODY,
    }


@pytest.fixture(scope="session")
def circular_text():
    return GCN_CIRCULAR


@pytest.fixture(scope="session")
def circular_msg():
    return models.GCNCircular.load(GCN_CIRCULAR)


@pytest.fixture(scope="session")
def avro_data_raw():
    return AVRO_DATA_EQUIVALENT


@pytest.fixture(scope="session")
def avro_data():
    return MESSAGE_AVRO


@pytest.fixture(scope="session")
def avro_msg():
    return models.AvroBlob.load(MESSAGE_AVRO)


@pytest.fixture(scope="session")
def voevent_fileobj():
    return io.BytesIO(VOEVENT_XML.encode())


@pytest.fixture(scope="session")
def voevent_text():
    return VOEVENT_XML


@pytest.fixture(scope="session")
def gcn_text_notice_fileobj():
    return io.BytesIO(MESSAGE_GCN_TEXT)


@pytest.fixture(scope="session")
def gcn_text_notice_data():
    return MESSAGE_GCN_TEXT


@pytest.fixture(scope="session")
def blob_text():
    return MESSAGE_BLOB


@pytest.fixture(scope="session")
def blob_msg():
    return {"content": MESSAGE_BLOB}


# avro_data needs to be loaded with single_record=False here because
message_parameters_dict_data = {
    # Generalize model_name, expected_model, test_file, and model_text
    # for easy access during tests. Useful when combined with parametrization
    # across message format, since fixtures (e.g., GCN_CIRCULAR) cannot be
    # used as parametrize arguments.
    "circular": {
        "model_name": "GCNCircular",
        "expected_model": models.GCNCircular,
        "test_file": "example_gcn.gcn3",
        "model_text": GCN_CIRCULAR,
    },
    "voevent": {
        "model_name": "VOEvent",
        "expected_model": models.VOEvent,
        "test_file": "example_voevent.xml",
        "model_text": VOEVENT_XML.encode(),
    },
    "gcntextnotice": {
        "model_name": "GCNTextNotice",
        "expected_model": models.GCNTextNotice,
        "test_file": "example_gcn.txt",
        "model_text": MESSAGE_GCN_TEXT,
    },
    "json": {
        "model_name": "JSONBlob",
        "expected_model": models.JSONBlob,
        "test_file": "example_json.json",
        "model_text": MESSAGE_JSON,
    },
    "blob": {
        "model_name": "Blob",
        "expected_model": models.Blob,
        "test_file": "example_blob.txt",
        "model_text": MESSAGE_BLOB,
    },
    "avro": {
        "model_name": "AvroBlob",
        "expected_model": models.AvroBlob,
        "test_file": "example_avro.avro",
        "model_text": MESSAGE_AVRO,
    },
    # -- Outdated versions --
    "voevent-encoded-as-json": {
        "model_name": "VOEvent",
        "expected_model": models.VOEvent,
        # no test file
        "model_text": VOEVENT_JSON,
    },
}


@pytest.fixture(scope="session")
def message_parameters_dict():
    return message_parameters_dict_data


@contextmanager
def temp_environ(**vars):
    """
    A simple context manager for temporarily setting environment variables

    Kwargs:
        variables to be set and their values

    Returns:
        None
    """
    from os import environ
    original = dict(environ)
    os.environ.update(vars)
    try:
        yield  # no value needed
    finally:
        # restore original data
        os.environ.clear()
        os.environ.update(original)


@contextmanager
def temp_config(tmpdir, data):
    """
    A context manager which creates a temporary config file with specified data

    Args:
        data: the data to be written to the file

    Returns:
        The path to the config directory for hop to use this config file, as a string
    """

    config_path = f"{tmpdir}/hop/config.toml"
    os.makedirs(os.path.dirname(config_path), exist_ok=True)
    config_file = open(config_path, mode='w')
    config_file.write(data)
    config_file.close()
    try:
        yield str(tmpdir)
    finally:
        # remove file
        os.remove(config_path)


@contextmanager
def temp_auth(tmpdir, data, perms=stat.S_IRUSR | stat.S_IWUSR):
    """
    A context manager which creates a temporary auth file with specified data and permissions

    Args:
        data: the data to be written to the file
        perms: the permissions which should be set on the file.
            The default value is to use the standard, safe permissions

    Returns:
        The path to the config directory for hop to use this config file, as a string
    """

    config_path = f"{tmpdir}/hop/auth.toml"
    os.makedirs(os.path.dirname(config_path), exist_ok=True)
    config_file = open(config_path, mode='w')
    os.chmod(config_path, perms)
    config_file.write(data)
    config_file.close()
    try:
        yield str(tmpdir)
    finally:
        # remove file
        os.remove(config_path)


class PhonyConnection:
    """A mock which pretends to be a urllib3.connection.HTTPConnection well enough to fool requests
    in simple situations, allowing testing HTTP connections without actually performing any network
    requests.
    """

    def __init__(self, responses):
        self.responses = responses
        self.counter = 0
        self.requests = []

    def urlopen(self, **kwargs):
        self.requests.append(kwargs)
        if self.counter >= len(self.responses):
            raise RuntimeError("urlopen called too many times on PhonyConnection")
        response = self.responses[self.counter]
        self.counter += 1
        response.url = kwargs["url"]
        return response


def mock_pool_manager(conn: PhonyConnection):
    """Produce a mock to stand in for urllib3.poolmanager.PoolManager which returns a pre-defined
    connection object.
    """
    pm = MagicMock()
    pm.connection_from_url = MagicMock(return_value=conn)
    pm.connection_from_host = MagicMock(return_value=conn)
    return MagicMock(return_value=pm)
