#!/usr/bin/python
#transport_udp_session.py
#
#
#    Copyright DataHaven.NET LTD. of Anguilla, 2006
#    Use of this software constitutes acceptance of the Terms of Use
#      http://datahaven.net/terms_of_use.html
#    All rights reserved.
#
#

"""
This is a part of transport_udp stuff.
Handle situations when that machine do have opened IP.
See `lib.transport_udp` for more details.

The firewall, anti-virus software and especially network routers may block incoming UDP traffic to you. 
In that case you need to take the initiative and start connecting to remote peer firstly.
So your NAT router will think that incoming UDP datagrams is just a responses and pass them to you.  
To be able to receive UDP datagrams from another machine both sides must act simultaneously, at the same time.
This code is doing that.
"""

import os
import sys
import time
import struct
import cStringIO


from twisted.internet import reactor


import dhnio
import settings
import misc
import identitycache
import tmpfile 
import nameurl
import contacts

import automat

#------------------------------------------------------------------------------ 

DHN_STUN_SERVERS = set([
    ('67.207.147.183', 13478),
    ('208.78.96.185', 13478),
    ('178.49.192.122', 13478),
    ])

DHN_STUN_COMMANDS = { 'ROUTE':       'r:',
                     }

#------------------------------------------------------------------------------ 

CODES = {'p:':  'PING',
         'g:':  'GREETING',
         'd:':  'DATA',
         'r:':  'REPORT',
         'a:':  'ALIVE',}

COMMANDS = {'PING':         'p:',
            'GREETING':     'g:',
            'DATA':         'd:',
            'REPORT':       'r:',
            'ALIVE':        'a:',}

BLOCK_SIZE_LEVELS = {   0: 508,
                        1: 1460,
                        2: 2048,
                        3: 3072,
                        4: 4096,
                        5: 5120,
                        6: 6144,
                        7: 7168,
                        8: 8124,
                        }
#sz = 508
#BLOCK_SIZE_LEVELS = {   #0: 508,
#                        0: sz,
#                        1: sz,
#                        2: sz,
#                        3: sz,
#                        4: sz,
#                        5: sz,
#                        6: sz,
#                        7: sz,
#                        }
MAX_BLOCK_SIZE_LEVEL = len(BLOCK_SIZE_LEVELS) - 1

BLOCK_RETRIES = 999999
MAX_WINDOW_SIZE = 32
MIN_WINDOW_SIZE = 1
MAX_ACK_TIME_OUT = 4.0
MIN_ACK_TIME_OUT = 0.5
MAX_TIMEOUTS_RATIO = 0.5
FINE_TIMEOUTS_RATIO = 0.0

#------------------------------------------------------------------------------ 

_ParentObject = None

_IdToAddress = {}
_AddressToId = {}

_SendStatusCallback = None
_ReceiveStatusCallback = None
_SendControlFunc = None
_ReceiveControlFunc = None
_RegisterTransferFunc = None
_UnRegisterTransferFunc = None
_StateChangedCallbackFunc = None
_NetworkAddressChangedFunc = None

_TimerPing = None
_TimerAlive = None
_Timer30sec = None

_SendingDelay = 0.01
_SendingTask = None

_OutboxQueueDelay = 0.01
_OutboxQueueTask = None

#------------------------------------------------------------------------------ 

def init(parent):
    global _ParentObject
    dhnio.Dprint(4, 'transport_udp_session.init')
    _ParentObject = parent
    reactor.callLater(0, process_outbox_queue)
    reactor.callLater(1, process_sessions)
    StartTimers()

   
def shutdown():
    dhnio.Dprint(4, 'transport_udp_session.shutdown')
    StopTimers()
    
   
def A(address, event=None, arg=None):
    s = open_session(address)
    if event is not None:
        s.automat(event, arg)
    return s
      
      
def parent():
    global _ParentObject
    return _ParentObject

#------------------------------------------------------------------------------ 

def dhn_stun_servers_enabled():
    return True

def dhn_stun_servers_request(transport, datagram):
    for addr in DHN_STUN_SERVERS:
        try:
            transport.write(datagram, addr)
        except:
            pass

def dhn_stun_server_response(datagram, address):
    try:
        myipport, peeridurl, peeripport, deltatime = datagram.split(' ')
        if peeripport == '0':
            return
        myip, myport = myipport.split(':')
        myport = int(myport)
        peerip, peerport = peeripport.split(':')
        peerport = int(peerport)
    except:
        dhnio.DprintException()    
        return False
    myaddress = (myip, myport)
    peeraddress = (peerip, peerport)
    x, ip, port, x = misc.getLocalIdentity().getProtoParts('udp')
    # if ip and port and ip != myip and str(port) != str(myport):
        # network_address_changed(myaddress)
        # return
    mappedaddress = peeraddress
    if identitycache.HasLocalIP(peeridurl):
        mappedaddress = (identitycache.GetLocalIP(peeridurl), peeraddress[1]) 
    if mappedaddress != peeraddress:
        if not is_session_opened(mappedaddress):
            s = open_session(mappedaddress)
            dhnio.Dprint(6, 'transport_udp_session.dhn_stun_server_response made a new LOCAL session %s for user [%s], address=%s' % (s.name, nameurl.GetName(peeridurl), str(mappedaddress)))
            s.event('init', peeridurl)
            s.routed = True
    else:
        if not is_session_opened(peeraddress):
            s = open_session(peeraddress)
            dhnio.Dprint(6, 'transport_udp_session.dhn_stun_server_response made a new session %s for user [%s]' % (s.name, nameurl.GetName(peeridurl)))
            s.event('init', peeridurl)
            s.routed = True

def network_address_changed(newaddress):        
    global _NetworkAddressChangedFunc
    if _NetworkAddressChangedFunc is not None:
        _NetworkAddressChangedFunc(newaddress)

#------------------------------------------------------------------------------ 
    
class OutboxFile():
    def __init__(self, peer, remote_idurl, file_id, filename, filesize, description=''):
        global _RegisterTransferFunc
        self.peer = peer
        self.remote_idurl = remote_idurl
        self.file_id = file_id
        self.filename = filename
        # self.transfer_id = transfer_id
        self.description = description
        self.size = filesize
        self.blocks = {} 
        self.num_blocks = 0
        self.bytes_sent = 0
        self.bytes_out = 0
        self.block_id = -1
        self.blocks_timeouts = 0
        self.blocks_counter = 0
        # self.failed_blocks = {}
        # self.has_failed_blocks = False
        self.started = time.time()
        self.timeout = max( int(self.size/settings.SendingSpeedLimit()), 5)
        self.transfer_id = None
        if _RegisterTransferFunc is not None:
            self.transfer_id = _RegisterTransferFunc('send', 
                self.peer.remote_address, self.get_bytes_sent, self.filename, self.size, self.description)
            # _RegisterTransferFunc(callback=self.get_bytes_sent, transfer_id=self.transfer_id)
        # dhnio.Dprint(6, 'transport_udp_session.OutboxFile [%d] to %s' % (self.file_id, nameurl.GetName(self.remote_idurl)))

    def close(self):
        global _UnRegisterTransferFunc
        if _UnRegisterTransferFunc and self.transfer_id:
            _UnRegisterTransferFunc(self.transfer_id)
        # dhnio.Dprint(6, 'transport_udp_session.OutboxFile close [%d] to %s (%s)' % (self.file_id, nameurl.GetName(self.remote_idurl), str(self.peer)))
        # dt = time.time() - self.started 
        # print 'DONE', self.bytes_sent, self.bytes_out, dt, (self.bytes_sent/1024.0)/dt 

    def get_bytes_sent(self):
        return self.bytes_sent 

    def report_block(self, block_id):
        if not self.blocks.has_key(block_id):
            dhnio.Dprint(10, 'transport_udp_session.report_block WARNING unknown block_id in REPORT packet received from %s: [%d]' % (self.peer.remote_address, block_id))
            return
        self.bytes_sent += len(self.blocks[block_id])
        del self.blocks[block_id]
        # sys.stdout.write('%d <<<\n' % block_id)

    def send(self):
        do_send = False
        while len(self.peer.sliding_window) < self.peer.sliding_window_size and self.block_id < self.num_blocks and self.block_id >= 0:
            if not self.send_block(self.block_id):
                break
            self.peer.sliding_window[(self.file_id, self.block_id)] = time.time()
            self.block_id += 1
            do_send = True
        return do_send
    
    def check_blocks_timeouts(self):
        if self.blocks_counter < 10:
            return 0
        ratio = float(self.blocks_timeouts) / float(self.blocks_counter)
        self.blocks_counter = 0
        self.blocks_timeouts = 0
        return ratio
        
    def send_block(self, block_id):
        global _SendControlFunc
        if not self.blocks.has_key(block_id):
            dhnio.Dprint(8, 'transport_udp_session.send_block WARNING block_id=%d not found, file_id=%s, transfer_id=%s, blocks: %d' % (
                block_id, self.file_id, str(self.transfer_id), len(self.blocks) ))
            return False
        data = self.blocks[block_id]
        if _SendControlFunc is not None:
            more_bytes = _SendControlFunc(self.peer.last_bytes_sent, len(data))
            if more_bytes < len(data):
                return False
        self.peer.last_bytes_sent = len(data)
        datagram = COMMANDS['DATA']
        datagram += struct.pack('i', self.file_id)
        datagram += struct.pack('i', block_id)
        datagram += struct.pack('i', self.num_blocks)
        datagram += struct.pack('i', len(data))
        datagram += data
        self.bytes_out += len(data)
        # sys.stdout.write('>>> %d\n' % block_id)
        return self.peer.sendDatagram(datagram)

#    def mark_failed_block(self, block_id):
#        if not self.failed_blocks.has_key(block_id):
#            self.failed_blocks[block_id] = 0
#        self.failed_blocks[block_id] += 1
#        if self.failed_blocks[block_id] >= BLOCK_RETRIES:
#            self.has_failed_blocks = True
#            dhnio.Dprint(14, 'transport_udp_session.mark_failed_block (%d,%d) is failed to send, so transfer %s is failed' % (self.file_id, block_id, self.transfer_id))
#            return True
#        return False
    
    def read_blocks(self):
        fin = open(self.filename, 'rb')
        block_id = 0
        while True:
            block_data = fin.read(BLOCK_SIZE_LEVELS[self.peer.block_size_level])
            if block_data == '':
                break
            self.blocks[block_id] = block_data
            block_id += 1  
        fin.close()
        self.num_blocks = block_id
        self.block_id = 0     
        
    def is_timed_out(self):
        return time.time() - self.started > self.timeout

    def cancel(self):
        dhnio.Dprint(6, 'transport_udp_session.OutboxFile cancel timeout=%d' % self.timeout)
        self.timeout = 0

#------------------------------------------------------------------------------ 

class InboxFile():
    def __init__(self, peer, file_id, num_blocks):
        global _RegisterTransferFunc
        self.fd, self.filename = tmpfile.make("udp-in")
        self.peer = peer
        self.file_id = file_id
        self.num_blocks = num_blocks
        self.blocks = {}
        self.bytes_received = 0
        self.bytes_extra = 0
        self.transfer_id = None
        self.started = time.time()
        self.last_block_time = time.time()
        self.block_timeout = 0
        if _RegisterTransferFunc is not None:
            self.transfer_id = _RegisterTransferFunc('receive', self.peer.remote_address, self.get_bytes_received, self.filename, -1, '')
        # dhnio.Dprint(6, 'transport_udp_session.InboxFile {%s} [%d] from %s' % (self.transfer_id, self.file_id, str(self.peer.remote_address)))

    def close(self):
        global _UnRegisterTransferFunc
        if _UnRegisterTransferFunc is not None and self.transfer_id is not None:
            _UnRegisterTransferFunc(self.transfer_id)
        try:
            if self.bytes_received > 0 and self.bytes_extra > self.bytes_received * 0.1:
                dhnio.Dprint(10, 'transport_udp_session.InboxFile.close WARNING %s%% garbage traffic from %s' % (str(self.bytes_extra/float(self.bytes_received)), self.peer.remote_address))
        except:
            dhnio.DprintException()
        # dhnio.Dprint(6, 'transport_udp_session.InboxFile.close {%s} [%d] from %s' % (self.transfer_id, self.file_id, str(self.peer.remote_address)))
        # dt = time.time() - self.started 
        # print 'DONE', self.bytes_received, self.bytes_extra, dt, (self.bytes_received/1024.0)/dt 

    def get_bytes_received(self):
        return self.bytes_received

    def input_block(self, block_id, block_data):
        if block_id not in self.blocks:
            self.blocks[block_id] = block_data
            self.bytes_received += len(block_data)
        else:
            self.bytes_extra += len(block_data)
        self.last_block_time = time.time()
        self.block_timeout = max( int(len(block_data)/settings.SendingSpeedLimit()), 3) 
    
    def build(self):
        for block_id in xrange(len(self.blocks)):
            os.write(self.fd, self.blocks[block_id])
        os.close(self.fd)
        # dhnio.Dprint(10, 'transport_udp_server.InboxFile.build [%s] file_id=%d, blocks=%d' % (
        #     os.path.basename(filename), file_id, len(self.incommingFiles[file_id])))

    def is_timed_out(self):
        if self.block_timeout == 0:
            return False
        return time.time() - self.last_block_time > self.block_timeout

    
#------------------------------------------------------------------------------ 

class TransportUDPSession(automat.Automat):
    """
    A state machine to connect users behind NAT, represent a session with one given remote peer. 
    See datahaven/documetns/automats.pdf - you can find a transtranport_udp_session page.
    """
    fast = True
    timers = {'timer-60sec':     (60,    ['PING',]),
              'timer-3min':      (60*10,  ['PING',]),
              }
    def __init__(self, address, name, state, debug_level):
        self.remote_address = address
        self.remote_idurl = None
        self.remote_name = None
        self.last_alive_packet_time = 0
        self.incommingFiles = {}
        self.outgoingFiles = {}
        self.maxOutgoingFiles = 1
        self.receivedFiles = {}
        self.outbox_queue = []
        self.tries = 0
        self.ack_timeout = MIN_ACK_TIME_OUT
        self.sliding_window = {}
        self.sliding_window_size = MIN_WINDOW_SIZE
        self.last_bytes_sent = 0
        # self.failedBlocks = {}
        self.hasFailedBlocks = False
        self.block_size_level = 0
        self.remote_id_request_last_time = 0
        self.routed = False
        automat.Automat.__init__(self, name, state, debug_level)
        
    def state_changed(self, oldstate, newstate):
        global _StateChangedCallbackFunc
        if _StateChangedCallbackFunc is not None:
            _StateChangedCallbackFunc(self.index, oldstate, newstate)
        
    def A(self, event, arg):
        #---CONNECTED---
        if self.state is 'CONNECTED':
            if event == 'datagram-received' :
                self.doReceiveData(arg)
            elif event == 'timer-30sec' and self.isAlive(arg) :
                self.doAlive(arg)
            elif event == 'shutdown' :
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'timer-30sec' and not self.isAlive(arg) :
                self.state = 'PING'
                self.doParentRequestRemoteID(arg)
                self.doPing(arg)
        #---AT_STARTUP---
        elif self.state is 'AT_STARTUP':
            if event == 'shutdown' :
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'init' :
                self.state = 'PING'
                self.doSaveIDURL(arg)
                self.doParentRequestRemoteID(arg)
                self.doPing(arg)
        #---PING---
        elif self.state is 'PING':
            if event == 'datagram-received' and self.isPingOrGreeting(arg) :
                self.state = 'GREETING'
                self.doReceiveData(arg)
                self.doGreeting(arg)
            elif event == 'datagram-received' and not self.isPingOrGreeting(arg) :
                self.doReceiveData(arg)
            elif event == 'shutdown' :
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'timer-60sec' :
                self.doParentRequestRemoteID(arg)
            elif event == 'timer-5sec' :
                self.doPing(arg)
            elif event == 'timer-3min' :
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
                self.doParentRecreateMe(arg)
        #---GREETING---
        elif self.state is 'GREETING':
            if event == 'datagram-received' and self.isGreetingOrAlive(arg) :
                self.state = 'CONNECTED'
                self.doReceiveData(arg)
                self.doAlive(arg)
            elif event == 'datagram-received' and not self.isGreetingOrAlive(arg) :
                self.doReceiveData(arg)
            elif event == 'timer-5sec' :
                self.doGreeting(arg)
            elif event == 'shutdown' :
                self.state = 'CLOSED'
                self.doDestroyMe(arg)
            elif event == 'timer-30sec' :
                self.state = 'PING'
                self.doParentRequestRemoteID(arg)
                self.doPing(arg)
        #---CLOSED---
        elif self.state is 'CLOSED':
            pass

    def isPingOrGreeting(self, arg):
        try:
            datagram, addr = arg
        except:
            return False
        return datagram[:2] in [ COMMANDS['PING'], COMMANDS['GREETING'], ] 
            
    def isGreetingOrAlive(self, arg):
        return arg[0][:2] in [ COMMANDS['GREETING'], COMMANDS['ALIVE'], ]
    
    def isAlive(self, arg):
        return time.time() - self.last_alive_packet_time < 60.0 * 1.0 + 5.0
    
    def doSaveIDURL(self, arg):
        self.remote_idurl = arg
        self.remote_name = nameurl.GetName(self.remote_idurl)
  
    def doPing(self, arg):
        self.sendDatagram(COMMANDS['PING'])
        if not self.routed:
            if dhn_stun_servers_enabled():
                if self.remote_idurl:
                    request = DHN_STUN_COMMANDS['ROUTE']+('%s %s' % (misc.getLocalID(), self.remote_idurl))
                    dhn_stun_servers_request(parent().client.transport, request)
    
    def doGreeting(self, arg):
        self.sendDatagram(COMMANDS['GREETING']+misc.getLocalID())
    
    def doAlive(self, arg):
        self.sendDatagram(COMMANDS['ALIVE'])
    
    def doReceiveData(self, arg):
        global _ReceiveControlFunc
        try:
            datagram, addr = arg
        except:
            return
        io = cStringIO.StringIO(datagram)
        code = io.read(2)
        cmd = CODES.get(code, None)
        if cmd is None:
            # dhnio.Dprint(2, 'transport_udp_session.doReceiveData WARNING incorrect data from %s: [%s]' % (self.remote_address, code))
            return
        if _ReceiveControlFunc:
            seconds_pause = _ReceiveControlFunc(len(datagram))
            if seconds_pause > 0:
                # transport_udp.A().client.transport.pauseProducing()
                parent().client.client.transport.pauseProducing()
                # self.client.transport.pauseProducing()
                # reactor.callLater(seconds_pause, transport_udp.A().client.transport.resumeProducing)
                reactor.callLater(seconds_pause, parent().client.transport.resumeProducing)
        # dhnio.Dprint(8, '                                UDP:[%s] from %s' % (cmd, self.remote_address))
        self.last_alive_packet_time = time.time()
        #--- DATA
        if cmd == 'DATA':
            try:
                file_id = struct.unpack('i', io.read(4))[0]
                block_id = struct.unpack('i', io.read(4))[0]
                num_blocks = struct.unpack('i', io.read(4))[0]
                data_size = struct.unpack('i', io.read(4))[0]
            except:
                dhnio.DprintException()
                return
            if not self.incommingFiles.has_key(file_id):
                if self.receivedFiles.has_key(file_id):
                    if time.time() - self.receivedFiles[file_id] > 10:
                        # we receive the file, but 10 seconds since that we still receive the acks
                        # definitely something wrong - drop session 
                        self.automat('shutdown')
                        return 
                    self.sendDatagram(COMMANDS['REPORT'] + struct.pack('i', file_id) + struct.pack('i', block_id))
                    return
                if len(self.incommingFiles) >= 2 * self.maxOutgoingFiles:
                    # too many incoming files - drop session
                    self.automat('shutdown') 
                    return
                self.incommingFiles[file_id] = InboxFile(self, file_id, num_blocks)
            inp_data = io.read()
            if len(inp_data) != data_size:
                dhnio.Dprint(2, 'transport_udp_session.doReceiveData WARNING incorrect datagram received from %s, not full data' % str(self.remote_address))
                return
            self.incommingFiles[file_id].input_block(block_id, inp_data) 
            self.sendDatagram(COMMANDS['REPORT'] + struct.pack('i', file_id) + struct.pack('i', block_id))
            # dhnio.Dprint(12, 'transport_udp_session.doReceiveData [%d] (%d/%d) from %s, state=%s' % (file_id, block_id, num_blocks, self.remote_address, self.state))
            if len(self.incommingFiles[file_id].blocks) == num_blocks:
                # dhnio.Dprint(12, 'transport_udp_session.doReceiveData new file [%d] (%d/%d) from %s' % (file_id, block_id, num_blocks, self.remote_address))
                self.incommingFiles[file_id].build()
                file_received(self.incommingFiles[file_id].filename, 'finished', 'udp', self.remote_address)
                self.incommingFiles[file_id].close()   
                del self.incommingFiles[file_id]   
                self.receivedFiles[file_id] = time.time()
                self.eraseOldFileIDs()
        #--- REPORT
        elif cmd == 'REPORT':
            try:
                file_id = struct.unpack('i', io.read(4))[0]
            except:
                dhnio.DprintException()
                return
            if not self.outgoingFiles.has_key(file_id):
                # if not self.receivedFiles.has_key(file_id):
                #     dhnio.Dprint(8, 'transport_udp_session.doReceiveData WARNING unknown file_id in REPORT packet received from %s: [%d]' % (self.remote_address, file_id))
                return
            try:
                block_id = struct.unpack('i', io.read(4))[0]
            except:
                dhnio.DprintException()
                return
            self.outgoingFiles[file_id].report_block(block_id)
            self.sliding_window.pop((file_id, block_id), None)
            self.sliding_window_size += 1
            if self.sliding_window_size > MAX_WINDOW_SIZE:
                self.sliding_window_size = MAX_WINDOW_SIZE
            if len(self.outgoingFiles[file_id].blocks) == 0:
                self.reportSentDone(file_id)
                self.closeOutboxFile(file_id)
            # dhnio.Dprint(8, 'transport_udp_session.doReceiveData REPORT on [%d,%d] received, blocks=%d, window=%d' % (file_id, block_id, len(self.outgoingFiles[file_id]), self.sliding_window_size))
        #--- GREETING
        elif cmd == 'GREETING':
            greeting_idurl = io.read()
            dhnio.Dprint(12, 'transport_udp_session.doReceiveData GREETING "%s" from %s, remote_idurl=%s, remote_address=%s, state=%s' % (
                greeting_idurl[:120], str(addr), self.remote_idurl, self.remote_address, self.state))
            if greeting_idurl:
                if nameurl.UrlMake(parts=nameurl.UrlParse(greeting_idurl)) != greeting_idurl:
                    dhnio.Dprint(2, 'transport_udp_session.doReceiveData WARNING incorrect idurl=%s in GREETING packet from %s' % (greeting_idurl, self.name))
                    self.sendDatagram(COMMANDS['PING'])
                else:
                    if self.remote_idurl is None:
                        self.remote_idurl = greeting_idurl
                        self.remote_name = nameurl.GetName(self.remote_idurl)
                        dhnio.Dprint(6, 'transport_udp_session.doReceiveData got idurl=%s in GREETING packet from %s' % (greeting_idurl, self.name))
                        parent().automat('recognize-remote-id', (self.index, self.remote_idurl))
                    else:
                        if self.remote_idurl != greeting_idurl:
                            dhnio.Dprint(2, 'transport_udp_session.doReceiveData WARNING wrong idurl=%s in GREETING packet from %s' % (greeting_idurl, self.name))
                    ident = contacts.getContact(greeting_idurl)
                    if ident:
                        udpip = ident.getIP('udp')
                        if udpip:
                            if udpip != addr[0]:
                                parent().automat('detect-remote-ip', (self.index, self.remote_idurl))
                    self.sendDatagram(COMMANDS['ALIVE'])
            else:
                dhnio.Dprint(2, 'transport_udp_session.doReceiveData WARNING did not found idurl in GREETING packet from %s, send PING again' % self.name)
                self.sendDatagram(COMMANDS['PING'])
        #--- PING
        elif cmd == 'PING':
            self.sendDatagram(COMMANDS['GREETING']+misc.getLocalID())
        #--- ALIVE            
        elif cmd == 'ALIVE':
            pass

    def doDestroyMe(self, arg):
        # dhnio.Dprint(6, 'transport_udp_session.doDestroyMe %s' % self.name)
        file_ids_to_remove = self.incommingFiles.keys()
        for file_id in file_ids_to_remove:
            try:
                os.close(self.incommingFiles[file_id].fd)
            except:
                dhnio.DprintException()
            file_received(self.incommingFiles[file_id].filename, 'failed', 'udp', self.remote_address, None, 'session has been closed')
            self.incommingFiles[file_id].close()   
            del self.incommingFiles[file_id]
        file_ids_to_remove = self.outgoingFiles.keys()
        for file_id in file_ids_to_remove:
            file_sent(self.remote_address, self.outgoingFiles[file_id].filename, 'failed', 'udp', None, 'session has been closed')
            self.closeOutboxFile(file_id)
        self.receivedFiles.clear()
        self.sliding_window.clear()
        automat.objects().pop(self.index)
        
    def doParentRequestRemoteID(self, arg):
        if self.remote_idurl:
            # dhnio.Dprint(8, 'transport_udp_session.doParentRequestRemoteID [%s] for session %s' % (self.remote_name, self.name))
            parent().automat('child-request-remote-id', self.remote_idurl)

    def doParentRecreateMe(self, arg):
        if self.remote_idurl:
            parent().automat('child-request-recreate', self.remote_idurl)


    def sendDatagram(self, data):
        # dhnio.Dprint(10, '    [UDP] %d bytes to %s (%s)' % (len(data), self.remote_address, nameurl.GetName(self.remote_idurl)))
        if parent().client is not None:
        # if transport_udp.A().client is not None:
            try:
                # transport_udp.A().client.transport.write(data, self.remote_address)
                parent().client.transport.write(data, self.remote_address)
                return True
            except:
                return False 
        else:
            dhnio.Dprint(2, 'transport_udp_session.sendDatagram WARNING client is None')
            return False      
        
    def makeFileID(self):
        return int(str(int(time.time() * 100.0))[4:])   

#    def register_outbox_file(self, filename, sz, callback, description=''): 
#        global _RegisterTransferFunc
#        if _RegisterTransferFunc is None:
#            return None
#        transfer_id = _RegisterTransferFunc('send', self.remote_address, callback, filename, sz, description)
#        return transfer_id
   
    def addOutboxFile(self, filename, description=''):
        self.outbox_queue.append((filename, description))
        # transfer_id, filesize = self.register_outbox_file(filename, description)
        # if transfer_id:
        #     self.outbox_queue.append((filename, filesize, transfer_id, description))
        # else:
        #     file_sent(self.remote_address, filename, 'failed', 'udp', None, 'unable to register transfer')
        # dhnio.Dprint(6, 'transport_udp_session.addOutboxFile [%s], total outbox files: %d' % (os.path.basename(filename), len(self.outbox_queue)))
        
    def putOutboxFile(self, filename, description=''):
        self.outbox_queue.insert(0, (filename, description))
        # transfer_id, filesize = self.register_outbox_file(filename, description)
        # if transfer_id:
        #     self.outbox_queue.insert(0, (filename, filesize, transfer_id, description)) 
        # else:
        #     file_sent(self.remote_address, filename, 'failed', 'udp', None, 'unable to register transfer')
        # dhnio.Dprint(6, 'transport_udp_session.putOutboxFile [%s], total outbox files: %d' % (os.path.basename(filename), len(self.outbox_queue)))
    
    def eraseOldFileIDs(self):
        if len(self.receivedFiles) > 10:
            file_ids = self.receivedFiles.keys()
            cur_tm = time.time()
            for file_id in file_ids:
                if cur_tm - self.receivedFiles[file_id] > 60 * 20:
                    del self.receivedFiles[file_id]
            del file_ids 

    def closeOutboxFile(self, file_id):
        self.outgoingFiles[file_id].close()
        del self.outgoingFiles[file_id]

    def reportSentDone(self, file_id):
        file_sent(self.remote_address, self.outgoingFiles[file_id].filename, 'finished', 'udp')
        self.block_size_level += 1
        if self.block_size_level > MAX_BLOCK_SIZE_LEVEL:
            self.block_size_level = MAX_BLOCK_SIZE_LEVEL
        # print self.block_size_level
    
    def reportSentFailed(self, file_id, why=''):
        file_sent(self.remote_address, self.outgoingFiles[file_id].filename, 'failed', 'udp', None, why)
        self.block_size_level -= 2
        if self.block_size_level < 0:
            self.block_size_level = 0
        # print self.block_size_level

    def processOutboxQueue(self):
        # global _UnRegisterTransferFunc
        has_reads = False
        while len(self.outbox_queue) > 0 and len(self.outgoingFiles) < self.maxOutgoingFiles:        
            # filename, filesize, transfer_id, description = self.outbox_queue.pop(0)
            filename, description = self.outbox_queue.pop(0)
            has_reads = True
            # we have a queue of files to be sent
            # somehow file may be removed before we start sending it
            # so we check it here and skip not existed files
            if not os.path.isfile(filename):
                file_sent(self.remote_address, filename, 'failed', 'udp', None, 'file were removed')
                # if _UnRegisterTransferFunc:
                #     _UnRegisterTransferFunc(transfer_id)
                continue
            try:
                sz = os.path.getsize(filename)
            except:
                file_sent(self.remote_address, filename, 'failed', 'udp', None, 'can not get file size')
                continue
#            if os.access(filename, os.R_OK):
#                file_sent(self.remote_address, filename, 'failed', 'udp', None, 'access denied')
#                continue
            file_id = self.makeFileID()
            f = OutboxFile(self, self.remote_idurl, file_id, filename, sz, description)
            f.read_blocks()
            self.outgoingFiles[file_id] = f
        return has_reads

    def checkIncomingFiles(self):
        for file_id in self.incommingFiles.keys():
            if self.incommingFiles[file_id].is_timed_out():
                self.automat('shutdown')
                return

    def processSending(self):
        has_sends = False
        has_timedout = False
        failed_ids = []
        for file_id in self.outgoingFiles.keys():
            if self.outgoingFiles[file_id].send():
                has_sends = True
            if self.outgoingFiles[file_id].is_timed_out():
                if self.outgoingFiles[file_id].timeout == 0:
                    failed_ids.append((file_id, 'canceled'))
                else:
                    # failed_ids.append((file_id, 'timeout'))
                    has_timedout = True
        for file_id, why in failed_ids:
            self.reportSentFailed(file_id, why)
            self.closeOutboxFile(file_id)
        del failed_ids
        if has_timedout:
            # if some packets we currently sending is timed out
            # all other in the outbox will fail too - so erase all.
            self.clearOutboxQueue()
        has_sends = has_sends or self.checkWindow()
        return has_sends
    
    def clearOutboxQueue(self):
        failed = []
        while len(self.outbox_queue) > 0:
            filename, filesize, transfer_id, description = self.outbox_queue.pop(0)
            failed.append((filename, filesize, transfer_id, description))
        for filename, filesize, transfer_id, description in failed:
            reactor.callLater(0, file_sent, self.remote_address, filename, 'failed', 'udp', None, 'timeout')
        del failed
            
    def checkWindow(self):
        if len(self.sliding_window) == 0:
            return False
        to_remove = [] 
        to_update = {}
        cur_tm = time.time()
        do_send = True
        for block_info, tm in self.sliding_window.items():
            if cur_tm - tm < self.ack_timeout: # give some time to get an Ack. 
                continue
            to_remove.append(block_info)
            file_id = block_info[0]
            block_id = block_info[1]
            if not self.outgoingFiles.has_key(file_id):
                # dhnio.Dprint(2, 'transport_udp_session.checkWindow WARNING unknown file_id [%d] from %s' % (file_id, nameurl.GetName(self.remote_idurl)))
                continue
            if not self.outgoingFiles[file_id].blocks.has_key(block_id):
                dhnio.Dprint(2, 'transport_udp_session.checkWindow WARNING unknown block_id: %d' % block_id)
                continue
            do_send = True
            to_update[block_info] = cur_tm
            outFile = self.outgoingFiles[file_id]
            outFile.blocks_timeouts += 1
            outFile.send_block(block_id)
            timeouts_ratio = outFile.check_blocks_timeouts()
            self.adjust_ack_timeout(timeouts_ratio)
#            self.outgoingFiles[file_id].send_block(block_id)
#            isFailed = self.outgoingFiles[file_id].mark_failed_block(block_id)
#            if not isFailed:
#                to_update[block_info] = cur_tm
        for block_info in to_remove:
            del self.sliding_window[block_info]
            # dhnio.Dprint(8, 'transport_udp_session.doCheckWindow (%d,%d) removed, window=%d' % (file_id, block_id, len(self.sliding_window)))
        for block_info, tm in to_update.items():
            self.sliding_window[block_info] = tm
            # dhnio.Dprint(8, 'transport_udp_session.doCheckWindow (%s) resending to %s, window=%d' % (block_info, self.remote_address, len(self.sliding_window)))
        if len(to_remove) > 0:
            self.sliding_window_size = int( self.sliding_window_size / 2.0 )
            if self.sliding_window_size < MIN_WINDOW_SIZE:
                self.sliding_window_size = MIN_WINDOW_SIZE
            # dhnio.Dprint(8, 'transport_udp_session.doCheckWindow decreased, window=%d' % self.sliding_window_size)
        del to_remove
        del to_update
        return 

    def adjust_ack_timeout(self, timeouts_ratio):
        if timeouts_ratio >= MAX_TIMEOUTS_RATIO:
            self.ack_timeout *= 2.0
            if self.ack_timeout > MAX_ACK_TIME_OUT:
                self.ack_timeout = MAX_ACK_TIME_OUT
                # TODO transfer is failed - udp packets did not get acket
                # drop connection!
                # TODO change transport_udp_server to drop connection
                self.automat('shutdown')
            # else:
            #     print('adjust_ack_timeout %f' % self.ack_timeout)
        elif timeouts_ratio <= FINE_TIMEOUTS_RATIO:
            self.ack_timeout /= 2.0
            if self.ack_timeout < MIN_ACK_TIME_OUT:
                self.ack_timeout = MIN_ACK_TIME_OUT
                # TODO transfer is great!
                #      increase window size, block size? 
            # else:
            #     print('adjust_ack_timeout %f' % self.ack_timeout)

    def cancel(self, transferID):
        for i in xrange(len(self.outbox_queue)):
            filename, filesize, transfer_id, description = self.outbox_queue[i]
            if transferID == transfer_id:
                del self.outbox_queue[i]
                break
        for file_id in self.outgoingFiles.keys():
            if self.outgoingFiles[file_id].transfer_id == transferID:
                self.outgoingFiles[file_id].cancel()
                # self.receivedFiles[file_id] = time.time()
                break
        for file_id in self.incommingFiles.keys():
            if self.incommingFiles[file_id].transfer_id == transferID:
                self.incommingFiles[file_id].build()
                file_received(self.incommingFiles[file_id].filename, 'failed', 'udp', self.remote_address)              
                self.incommingFiles[file_id].close()
                del self.incommingFiles[file_id]
                self.receivedFiles[file_id] = time.time()
                break

#------------------------------------------------------------------------------ 
    
def check_outbox_queue():    
    has_reads = False
    for sess in sessions():
        if sess.processOutboxQueue():
            has_reads = True
    if has_reads:
        check_sending_queue()
    return has_reads 
    
def check_sending_queue():    
    has_sends = False
    for sess in sessions():
        if sess.processSending():
            has_sends = True
    return has_sends

def check_incoming_files():
    for sess in sessions():
        sess.checkIncomingFiles()
    
def process_outbox_queue():
    global _OutboxQueueDelay
    global _OutboxQueueTask
    
    has_reads = check_outbox_queue()

    _OutboxQueueDelay = misc.LoopAttenuation(
        _OutboxQueueDelay, 
        has_reads, 
        settings.MinimumSendingDelay(), 
        settings.MaximumSendingDelay(),)
    
    # attenuation
    _OutboxQueueTask = reactor.callLater(_OutboxQueueDelay, process_outbox_queue)
        

def process_sessions():
    global _SendingDelay
    global _SendingTask
    
    check_incoming_files()
    
    has_sends = check_sending_queue()
    
    _SendingDelay = misc.LoopAttenuation(
        _SendingDelay, 
        has_sends, 
        settings.MinimumSendingDelay(), 
        settings.MaximumSendingDelay(),)
    
    # attenuation
    _SendingTask = reactor.callLater(_SendingDelay, process_sessions)
    
#------------------------------------------------------------------------------ 

def sessions():
    for a in automat.objects().values():
        if a.name.startswith('udp_'):
            yield a


def open_session(address):
    name = 'udp_%s_%s' % (address[0].replace('.','_'), str(address[1]))
    for a in automat.objects().values():
        if a.name == name:
            return a
    a = TransportUDPSession(address, name, 'AT_STARTUP', 12)
    return a


def get_session(address):
    name = 'udp_%s_%s' % (address[0].replace('.','_'), str(address[1]))
    for a in automat.objects().values():
        if a.name == name:
            return a
    return None


def get_session_by_index(index):
    return automat.objects().get(index, None)


def is_session_opened(address):
    name = 'udp_%s_%s' % (address[0].replace('.','_'), str(address[1]))
    for a in automat.objects().values():
        if a.name == name:
            return True
    return False
        

def outbox_file(address, filename, fast=False, description=''):
#    dhnio.Dprint(6, 'transport_udp_session.outbox_file %s to %s (%s)' % (
#        os.path.basename(filename), nameurl.GetName(idurl), str(address)))
    s = open_session(address)
    if s.state != 'CONNECTED':
        file_sent(s.remote_address, filename, 'failed', 'udp', None, 'remote peer is not connected')
        return
    if fast:
        s.putOutboxFile(filename, description)
    else:
        s.addOutboxFile(filename, description)
    check_outbox_queue()


def cancel(transferID):
    for sess in sessions():
        sess.cancel(transferID)
    

def big_event(event, arg=None):
    for sess in sessions():
        sess.event(event, arg)


def data_received(datagram, address):
    try:
        from transport_control import black_IPs_dict
        if address[0] in black_IPs_dict().keys() or '.'.join(address[0].split('.')[:-1]) in black_IPs_dict().keys():
            # dhnio.Dprint(12, 'transport_udp_session.data_received %d bytes from BLACK IP: %s, %s' % (len(datagram), str(address[0]), str(black_IPs_dict()[address[0]])) )
            return
    except:
        pass
    if dhn_stun_servers_enabled():
        if address in DHN_STUN_SERVERS:
            dhn_stun_server_response(datagram, address)
            return
    if not is_session_opened(address):
        s = open_session(address)
        idurl = identitycache.GetIDURLByIPPort(address[0], address[1])
        if idurl:
            dhnio.Dprint(6, 'transport_udp_session.data_received from KNOWN user, made a new session %s for [%s]' % (s.name, nameurl.GetName(idurl)))
            s.event('init', idurl)
        else:
            dhnio.Dprint(6, 'transport_udp_session.data_received from UNKNOWN user, made a new session %s' % s.name)
            s.event('init', None)
    A(address, 'datagram-received', (datagram, address))


def shutdown_all_sessions():
    big_event('shutdown')
    
    
def file_received(filename, status, proto='', host=None, error=None, message=''):
    global _ReceiveStatusCallback
    if _ReceiveStatusCallback is not None:
        _ReceiveStatusCallback(filename, status, proto, host, error, message)
    
    
def file_sent(host, filename, status, proto='', error=None, message=''):
    global _SendStatusCallback
    if _SendStatusCallback is not None:
        _SendStatusCallback(host, filename, status, proto, error, message)


#def timer30sec():
#    global _Timer30sec
#    big_event('timer-30sec')
#    _Timer30sec = reactor.callLater(ndom.randint(25,35), timer30sec)


def timerAlive():
    global _TimerAlive
    big_event('timer-30sec')
    _TimerAlive = reactor.callLater(30, timerAlive)


def timerPing():
    global _TimerPing
    big_event('timer-5sec')
    _TimerPing = reactor.callLater(5, timerPing)
    
#------------------------------------------------------------------------------ 
    
def SetReceiveStatusCallback(cb):
    global _ReceiveStatusCallback
    _ReceiveStatusCallback = cb
    
def SetSendStatusCallback(cb):
    global _SendStatusCallback
    _SendStatusCallback = cb
    
def SetSendControlFunc(f):
    global _SendControlFunc
    _SendControlFunc = f
    
def SetReceiveControlFunc(f):
    global _ReceiveControlFunc
    _ReceiveControlFunc = f

def SetRegisterTransferFunc(f):
    global _RegisterTransferFunc
    _RegisterTransferFunc = f

def SetUnRegisterTransferFunc(f):
    global _UnRegisterTransferFunc
    _UnRegisterTransferFunc = f

def SetStateChangedCallbackFunc(f):
    global _StateChangedCallbackFunc
    _StateChangedCallbackFunc = f
    
def SetNetworkAddressChangedFunc(f):
    global _NetworkAddressChangedFunc
    _NetworkAddressChangedFunc = f
    
#------------------------------------------------------------------------------ 

    
def StartTimers():
    global _Timer30sec
    global _TimerAlive
    global _TimerPing
#    if _Timer30sec is None:
#        timer30sec()
    if _TimerAlive is None: 
        timerAlive()
    if _TimerPing is None:
        timerPing()


def StopTimers():
    global _Timer30sec
    global _TimerAlive
    global _TimerPing
#    if _Timer30sec:
#        _Timer30sec.cancel()
#        _Timer30sec = None
    if _TimerAlive:
        _TimerAlive.cancel()
        _TimerAlive = None
    if _TimerPing:
        _TimerPing.cancel()
        _TimerPing = None





