# Martin Konečnik, http://git.siwim.si/mkonecnik/integration-dsbt
# Script handles receiving vehicle information from SiWIM-I, converting it to json and sending it to DSBT.
import argparse
import imghdr
import json
import os
import re
import socket
import sys
import time
from configparser import ConfigParser
from datetime import datetime
from logging import CRITICAL, DEBUG, ERROR, Formatter, INFO, StreamHandler, WARNING, getLogger
from shutil import copyfile, move
from threading import Thread
from time import gmtime, strftime
from typing import Optional
from uuid import uuid4
from warnings import filterwarnings

import requests
from cestel_console.helpers import configure_all
from lxml import etree
from py_logging.cestel_logging import init_logger, log, log_exception
from pycountry import countries

sys.path.append('..')
import helpers

filterwarnings("ignore", category=DeprecationWarning)


# Method constructs the image path and returns it (or None if no image is found).
def get_image_path(img_type: str, ts: str, lane: int) -> Optional[str]:
    logger_sending = getLogger('sending')
    file_name = os.path.join(SITE_DIR, str(PHOTOS_DICT[f'{img_type}_{lane}']), f'{ts[:10]}-hh-mm-ss', f'{ts[:13]}-mm-ss', f'{ts}_{lane}_p00.jpg')
    log(logger_sending, f'Full image filename: {file_name}', sev=DEBUG)
    if os.path.exists(file_name):
        typ = imghdr.what(file_name)
        if typ != 'jpeg':
            log(logger_sending, f'Image {file_name} is not a valid jpeg. Detected type: {typ}.', sev=DEBUG)
            return None
        return file_name
    else:
        log(logger_sending, f'Image {file_name} not found.', sev=DEBUG)
        return None


def get_veh_type(letter: str) -> str:
    if letter == 'C':
        return '1'
    elif letter == 'L' or letter == 'H':
        return '2'
    elif letter == 'B':
        return '3'
    elif letter == 'M':
        return '4'
    else:
        return '0'


def dump_package(directory: str, files: dict, exist_ok: bool = False, headers: dict = None, error: str = str()) -> None:
    """ Saves the package to disk
    :param directory: Directory that we want to save to.
    :param files: Dictionary of tuples containing file name and file data.
    :param exist_ok: Passed to os.makedirs.
    :param headers: If passed, headers are written to info.txt.
    :param error: Error message that was received.
    """
    os.makedirs(directory, exist_ok=exist_ok)
    if headers:
        with open(os.path.join(directory, 'info.txt'), 'w') as f:
            f.write(f'{headers}\n')
            f.write(error)
    for fname, fdata in files.items():
        with open(os.path.join(directory, fname), 'wb') as f:
            f.write(fdata[1])


def send_files(msg: str, conf: dict, veh_classes: dict, defaults: dict, log_level: int, local=False) -> None:
    # The logger must be initialised in the thread it's logging in or style will be lost (naturally it can be done otherwise, but not worth it here).
    sending_logger = getLogger('sending')
    console_logger = getLogger('info')
    json_dict = process_message(msg, conf, veh_classes, defaults.copy(), local=local)  # We have to pass a copy of the defaults dictionary, or we'll just be adding to it constantly.
    if json_dict:
        ts = datetime.strptime(json_dict['datetime'][:-6], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
        headers = server_headers.copy()  # Make a copy of the default dict.
        headers['Api-Request-Datetime'] = f'{datetime.now():%Y-%m-%dT%H:%M:%S}{strftime("%z", gmtime())}'  # Datetime in format '2021-03-26T16:04:08+0100'

        files = dict()
        for image in json_dict['media']:
            typ = image['type']
            name = image['name']
            files[f'{typ}-{name}'] = (f'{typ}-{name}', open(os.path.join(image['baseurl'], name), 'rb').read(), 'application/octet-stream')

        for i in range(len(json_dict['media'])):  # Set all baseurls to empty string and fix names.
            json_dict['media'][i]['baseurl'] = str()
            json_dict['media'][i]['name'] = f'{json_dict["media"][i]["type"]}-{json_dict["media"][i]["name"]}'  # Change the name variable, to reflect the key value.

        if DO_SIGN:
            signature = sign_data(json.dumps(json_dict, ensure_ascii=False).encode())

            if signature is None:
                log(sending_logger, f'Signing failed! See signing log for more info. Dumping {ts} to {FAILED_DIR}.', sev=ERROR)
                with open(os.path.join(FAILED_DIR, f'{ts}.json'), 'w') as f:
                    json.dump(json_dict, f)
                return
        else:
            signature = bytes()

        if encrypt_cert:
            files[f'{ts}.crt'] = (f'{ts}.crt', encrypt_cert, 'application/octet-stream')
        else:
            files[f'{ts}.crt'] = (f'{ts}.crt', bytes(), 'application/octet-stream')
        if signature:
            files[f'{ts}.sign'] = (f'{ts}.sign', signature, 'application/octet-stream')
        else:
            files[f'{ts}.sign'] = (f'{ts}.sign', bytes(), 'application/octet-stream')
        files[f'{ts}.json'] = (f'{ts}.json', json.dumps(json_dict, ensure_ascii=False).encode(), 'application/json')  # Add json last, since baseurl has to be cleared first.

        if NO_SEND or log_level <= DEBUG:
            dump_package(os.path.join(DEBUG_PACKAGE_DIR, ts), files, exist_ok=True)

        if NO_SEND:  # Don't send if running test.
            log(sending_logger, 'Not sending anything, because NO_SEND is set. This gets set automatically if running with --local or --ip parameters.', sev=WARNING)
            return

        retry = 0
        while retry < MAX_RETRIES:
            try:
                headers['Api-Request-Uuid'] = str(uuid4())  # Each request must have a unique identifier.
                r = requests.post(server_url, headers=headers, files=files)
            except:
                console_logger.warning(f'Server did not respond for {ts}. Retries left {MAX_RETRIES - retry - 1}')
                retry += 1
                time.sleep(1)
                continue
            if r.status_code == 200:
                response = r.json()
                if response['status'] != 0:  # If status is not 0 and error has occurred and should be noted and event saved for debugging.
                    log(sending_logger, f'Failed to send {ts}: {r.text}. Saving package to {ERROR_DIR}', sev=ERROR)
                    dump_package(os.path.join(ERROR_DIR, ts), files, headers=headers, error=r.text)  # If we get a non-zero response, something is wrong so we dump the files.
                else:
                    console_logger.info(f'Vehicle {ts} sent successfully.')  # This logger isn't created with cestel_logging, so can't use the soon deprecated log method.
                    log(sending_logger, f'Server responded with: {response}')
                    if log_level <= DEBUG:
                        with open(os.path.join(SUCCEEDED_DIR, ts + '.json'), 'w') as outfile:
                            json.dump(json_dict, outfile)
                break
            log(sending_logger, f'Sending attempt failed with status code {r.status_code} and response {r.text}', sev=DEBUG)
            retry += 1
            time.sleep(1)
        else:
            log(sending_logger, f'Failed to send vehicle {ts} after {retry} attempts!', sev=WARNING)
            dump_package(os.path.join(FAILED_DIR, ts), files)
        log(sending_logger, f'Sending process for {json_dict["datetime"]} exited.')


def setup_signing_interface(password: bytes) -> tuple:  # Returns tuple of EUGetInterface and info
    start = time.time()
    logger_sign = getLogger('signing')
    EULoad()
    log(logger_sign, f'Load time: {round(time.time() - start, 4)}s', sev=DEBUG)
    pIface = EUGetInterface()
    try:
        pIface.SetUIMode(False)
        pIface.Initialize()
        pIface.SetUIMode(False)
    except Exception as e:
        log(logger_sign, f'Initialize failed: {e}', sev=ERROR)
        EUUnload()
        return None, None

    log(logger_sign, 'Library Initialized', sev=DEBUG)

    dwType = 0
    lDescription = []
    try:
        pIface.EnumKeyMediaTypes(dwType, lDescription)
        while lDescription[0] != 'е.ключ ІІТ Алмаз-1К':
            dwType += 1
            if not pIface.EnumKeyMediaTypes(dwType, lDescription):
                log(logger_sign, 'KeyMedia not found', sev=ERROR)
                pIface.Finalize()
                EUUnload()
                return None, None

    except Exception as e:
        log(logger_sign, f'EnumKeyMediaTypes failed: {e}', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None

    lDescription = []
    if not pIface.EnumKeyMediaDevices(dwType, 0, lDescription):
        log(logger_sign, 'Device not found', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None
    log(logger_sign, f'Device description: {lDescription[0]}', sev=DEBUG)

    pKM = {'szPassword': password, 'dwDevIndex': 0, 'dwTypeIndex': dwType}
    try:
        pInfo = dict()  # Info about certificate is stored here.
        pIface.ReadPrivateKey(pKM, pInfo)
    except Exception as e:
        log(logger_sign, f'Key reading failed: {e}', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None

    log(logger_sign, 'Key read successfully.', sev=DEBUG)
    return pIface, pInfo


def create_limits(veh: etree.Element, veh_classes: dict) -> list:
    """ Builds the limits list. """
    limits = list()

    veh_class = veh.find('cls').text
    if veh_class == '140':  # If it's a 140 ... can't use classification table, so just give up.
        return limits

    # Add the weight limit.
    limits.append(
        {
            'key': 'weight-limit',
            'value': str(helpers.convert_kN_to_kg(veh_classes[f'subclass_{veh_class}']['max_GVW__kN']))
        }
    )

    limit_list = veh_classes[f'subclass_{veh_class}']['max_axle_weight__kN'].split(';')
    limits.append(
        {
            'key': 'axleload-limit',
            'value': str(helpers.convert_kN_to_kg(limit_list[0].split(',')[1]))
        }
    )

    axgrps = veh.find('axgrps').text
    if axgrps != len(axgrps) * '1':  # If axgrps string isn't all single axles, we have to set limit for groups.
        limit_list.reverse()  # Groups are at the end, so search that way.
        first_axle = re.search('[^1]', axgrps).start() + 1  # Get the first axle of the first group  # TODO Do something for multiple
        for limit in limit_list:
            if limit.find('+') != -1 and limit.split('+')[0] == str(first_axle):
                limits.append(
                    {
                        'key': 'groupload-limit',
                        'value': str(helpers.convert_kN_to_kg(limit.split(',')[1]))
                    }
                )

    return limits


def create_measurements(veh: etree.Element) -> list:
    """ Builds the measurements list. """
    # noinspection PyTypeChecker
    measures = [
        {
            'key': 'speed-main',
            'value': str(helpers.convert_m_per_s_to_km_per_h(veh.find('v').text))
        },
        {
            'key': 'weight-full',
            'value': str(helpers.convert_kN_to_kg(veh.find('gvw').text))
        },
        {
            'key': 'length',
            # 0 until they decide otherwise
            'value': str(0)
            #'value': str(convert_m_to_mm(sum([float(el.text) for el in veh.xpath('ads/d')])))
        },
        {
            'key': 'axlecount',
            'value': veh.find('naxles').text
        },
        {
            'key': 'temperature',
            'value': veh.find('T').text
        },
        {
            'key': 'overweighted',
            # If either axle or total weight is too high, set to true otherwise to false.
            'value': '1' if (veh.find('aol') is not None or veh.find('vol') is not None) else '0'
        },
        {
            'key': 'oversized',
            # 0 until they decide otherwise
            'value': str(0)
        }
    ]

    single_load = list()  # List of axle weights.
    group_load = list()  # List of group weights.
    count = 0  # Index in weights list.
    axle_count = 1  # Index of axle/group.
    group_count = 1
    weights = [float(el.text) for el in veh.xpath('acws/w')]
    for weight in weights:
        single_load.append(
            {
                'key': f'axleload-{axle_count}',
                'value': str(helpers.convert_kN_to_kg(weight))
            }
        )
        axle_count += 1
    for axle in veh.find('axgrps').text:  # Loop over the axgrps string to determine which weights are groups.
        if axle != '1':  # Handle non-single axles.
            group_load.append(
                {
                    'key': f'groupload-{group_count}',
                    'value': str(helpers.convert_kN_to_kg(sum(weights[count:count + int(axle, 16)])))
                }
            )
            group_count += 1
        count += int(axle, 16)

    axle_count = 1
    axle_distances = list()  # List of axle distances.
    for d in veh.xpath('ads/d'):
        axle_distances.append(
            {
                'key': f'axledistance-{axle_count}',
                'value': str(helpers.convert_m_to_mm(d.text))
            }
        )
        axle_count += 1

    return measures + single_load + group_load + axle_distances  # Concatenate the lists.


def get_image_dict(image_type: str, ts: str, lane: int, tz: str, use_image_time: bool = False, local: bool = False) -> Optional[dict]:
    send_logger = getLogger('sending')
    log(send_logger, f'Looking for {image_type}', sev=DEBUG)
    image_path = get_image_path(image_type, ts, lane)
    if image_path:
        log(send_logger, f'Found {image_type} image: {image_path}', sev=DEBUG)
        if local:  # Copy the images if running with local parameter.
            copyfile(image_path, f'{os.path.join(DEBUG_PACKAGE_DIR, image_type)}.jpg')
        if DO_SIGN:
            img_hash = list()
            try:
                pIface.HashFile(image_path, None, img_hash)  # Saves the hash into img_hash
                log(getLogger('signing'), f'Hash calculated for {image_type} image {os.path.basename(image_path)} {helpers.to_hex(img_hash[0])}.')
            except RuntimeError as e:
                log(getLogger('signing'), f'An error has occurred when trying to calculate hash for {image_type} image {os.path.basename(image_path)}. Hash is {helpers.to_hex(img_hash[0])}. Error: {e}', sev=ERROR)
                return None
        return {
            'type': image_type,
            'baseurl': os.path.dirname(image_path),  # Used for sending images. Should be set to empty string once done.
            'name': os.path.basename(image_path),
            'hash': [
                'gost34311',
                helpers.to_hex(img_hash[0]) if DO_SIGN else '35dd5daf62e62e6e3a3f2c6fbbc47d96bf3e0c5d02f3fc00adf7683af172d78f'  # If signing is not enabled, we don't try to calculate hash either.
            ],
            'datetime': helpers.get_photo_ts(image_path, tz) if use_image_time else helpers.convert_ts(ts, tz)
        }


def sign_data(json_file: bytes):  # Returns signature or None if it fails.
    logger_sign = getLogger('signing')

    start = time.time()
    signature = list()
    try:
        pIface.SignData(json_file, len(json_file), None, signature)
    except Exception as e:
        log(logger_sign, f'SignData failed: {e}', sev=ERROR)
        return None

    log(logger_sign, f'Data sign done in {round(time.time() - start, 2)}s')
    return signature[0]


def add_photo(img_dict: dict, img_type: str, media: list, ts: str, lane: int, tz: str, local: bool = False) -> None:
    if img_dict[img_type] is None:  # If plate hasn't been found yet.
        img_dict[img_type] = get_image_dict(img_type, ts, lane, tz, local=local)  # Check if image exists.
        if img_dict[img_type]:  # If image was found, append the data.
            media.append(img_dict[img_type])


def process_message(msg: str, conf: dict, veh_classes: dict, json_dict: dict, local: bool = False) -> Optional[dict]:
    send_logger = getLogger('sending')
    try:
        xml = etree.fromstring(msg)
    except etree.XMLSyntaxError:
        log(send_logger, f'Failed to parse string {msg}', sev=ERROR)
        return None

    event = xml.xpath('site/vehicles/vehicle')[0]
    veh = event.find('wim')
    anpr = event.find('anpr')
    siwim_ts = veh.find('ts').text
    lane_num = int(veh.find('lane').text)

    log(send_logger, f'Processing for {siwim_ts} started.')

    # Process flags
    flags = veh.find('flags').text
    eur6 = helpers.classify_eur6(veh.find('cls').text)
    if eur6 <= 3 or helpers.check_flag(flags, 1, 2) or helpers.check_flag(flags, 0, 8):  # Discard vehicle if it has set disregard flag or is classified as 1, 2 or 3.
        getLogger('info').info(f'Vehicle {siwim_ts} with flags {flags} and eur6 class {eur6} discarded.')
        return None
    
    # They're only interested in overloaded vehicles
    #if veh.find('aol') == None and veh.find('vol') == None:
    #  getLogger('info').info(f'Vehicle {siwim_ts} is not overloaded and is therefor -- discarded.')
    #  return None

    # datetime
    json_dict['uuid'] = str(uuid4())

    timezone = int(veh.find('tzbias').text) / 3600
    if not timezone.is_integer():
        log(send_logger, f'Adding timezone information to string is implemented only for integer values. Timezone {timezone} is not supported.', sev=CRITICAL)
    timezone_str = f'0{int(timezone)}:00' if timezone < 10 else f'{timezone}:00'
    dsbt_ts = helpers.convert_ts(siwim_ts, timezone_str)
    json_dict['datetime'] = dsbt_ts

    # noinspection PyTypeChecker
    json_dict['limits'] = create_limits(veh, veh_classes)

    # noinspection PyTypeChecker
    json_dict['measurements'] = create_measurements(veh)

    # vehicle
    json_dict['vehicle'] = dict()
    if anpr is not None:
        # noinspection PyTypeChecker
        json_dict['vehicle']['plate'] = [
            {
                'text': anpr.find('lp').text if anpr.find('lp') is not None else '',
                'placement': anpr.find('location').text if anpr.find('location') is not None else 'front'
            }
        ]
        try:
            json_dict['vehicle']['plate'][0]['country'] = countries.get(alpha_3=anpr.find('country').text).numeric  # Use pycountry to get the country.
        except Exception as e:  # Just catch any exception, since if we can't retrieve the country it's not that big of a deal.
            log(send_logger, f'Failed to retrieve country: {e}', sev=DEBUG)
    else:
        # noinspection PyTypeChecker
        json_dict['vehicle']['plate'] = [
            {
                'text': 'notrecognized',
                'placement': 'front'
            }
        ]
    axcfg = veh.find('axconfig').text if veh.find('axconfig') is not None else ''
    # noinspection PyTypeChecker
    json_dict['vehicle']['params'] = [
        {
            'key': 'type',
            'value': get_veh_type(axcfg[0]) if axcfg != str() and axcfg is not None else '0'
        },
        {
            'key': 'class',
            'value': str(helpers.classify_ua(veh.find('cls').text))
        },
        {
            'key': 'color',
            'value': '000'
        },
        {
            'key': 'direction',
            'value': conf['Directions'][f'lane_{lane_num}']
        },
        {
            'key': 'mark',
            'value': 'unknown'
        },
        {
            'key': 'model',
            'value': 'unknown'
        },
        {
            'key': 'lane',
            'value': str(lane_num + int(conf['General'].get('lane_offset', '-1')))
        }
    ]

    img_dict = {
        'plate': None,
        'front': None,
        'side': None
    }
    json_dict['media'] = list()
    start = time.time()
    while time.time() - start < args.timeout:  # Does this make sense here? Couldn't I just add the filenames here and worry if they exist or not when sending data?
        # Add plate photo
        add_photo(img_dict, 'plate', json_dict['media'], siwim_ts, lane_num, timezone_str, local=local)
        # Add front photo
        add_photo(img_dict, 'front', json_dict['media'], siwim_ts, lane_num, timezone_str, local=local)
        # Add side photo
        add_photo(img_dict, 'side', json_dict['media'], siwim_ts, lane_num, timezone_str, local=local)
        log(send_logger, f'Found {len(json_dict["media"])} images.', sev=DEBUG)
        if len(json_dict['media']) == 3:
            log(send_logger, f'Found all images.', sev=DEBUG)
            break
        time.sleep(1)
    helpers.are_images_present(img_dict, siwim_ts)  # Checks if we found all images and prints warnings if that's not the case.
    log(send_logger, f'Processing for {siwim_ts} stopped.')
    return json_dict


if __name__ == '__main__':
    APP_NAME = 'Converter for DSBT v1.1.0'
    DEBUG_DIR = 'debug'
    DEBUG_PACKAGE_DIR = 'package'
    FAILED_DIR = 'failed_to_send'
    ERROR_DIR = 'error'
    SUCCEEDED_DIR = 'succeeded_sending'
    CERT_DIR = '../certificate'
    MAX_RETRIES = 5
    server_url = 'https://wcs-data.dsbt.gov.ua/api/pkd/wimpack'
    server_headers = {
        'Api-Authentication': '2223343543827821',
    }

    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--conf', help='path to the configuration file', default=r'..\ua.conf', type=str)
    parser.add_argument('--conf_i', help='path to SiWIM-I configuration file', default=r'D:\siwim_mkiii\siwim_i\conf\conf.xml', type=str)
    parser.add_argument('-d', '--debug', help='run application with debug output', nargs='?', const=DEBUG, default=WARNING, type=int)
    parser.add_argument('-i', '--ip', help=f'IP of I to which we\'re connecting; disables sending of files', default='127.0.0.1', type=str)
    parser.add_argument('-l', '--local', help='define local xml to process; disables sending of files', type=str)
    parser.add_argument('--memory', help=f'set memory limit for the application (0 to disable)', default=100, type=int)
    parser.add_argument('--restart', help='this flag is passed to indicate the application was restarted automatically (due to running out of memory)', action='store_true')
    parser.add_argument('-t', '--timeout', help=f'how long to wait for images before deciding they don\'t exist', default=10, type=int)
    parser.add_argument('-v', '--classes', help=f'path to vehicle_classes.conf, if not passed it will be determined using E\'s conf file', type=str)
    args = parser.parse_args()

    if args.local or parser.get_default('ip') != args.ip:  # Disable sending if we're processing a local file or receiving data from a non-localhost IP.
        NO_SEND = True
        SITE_DIR = str()  # If running locally, the whole image path has to be passed.
    else:
        NO_SEND = False
        SITE_DIR = r'D:\siwim_mkiii\sites'  # When running on site, partial path will be deduced based on E's conf.
    DO_SIGN = False

    logger_main = init_logger('main', to_console=True, console_level=INFO if args.debug > INFO else args.debug, level=INFO if args.debug > INFO else args.debug)  # Log DEBUG if debugging, otherwise INFO.
    logger_sending = init_logger('sending', to_console=True, console_level=args.debug, level=args.debug)
    init_logger('signing', level=args.debug, to_console=True, console_level=args.debug)
    # Logger that's logging to console only isn't implemented in py_logging 1.0 yet.
    logger_info = getLogger('info')
    console = StreamHandler()
    console.setLevel(INFO)
    console.setFormatter(Formatter(fmt='%(name)-9s %(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
    logger_info.addHandler(console)
    logger_info.setLevel(INFO)

    try:
        conf = ConfigParser()
        conf.read(args.conf, encoding='utf-8-sig')
        ID = conf['General']['site_uuid']
        SITE_ID = conf['General']['site_id']
        ADDRESS = conf['General']['address']
        SERIAL = conf['General']['serial']
        LATITUDE = float(conf['General']['latitude'])
        LONGITUDE = float(conf['General']['longitude'])
        DIRECTIONS = dict(conf['Directions'])
    except:
        log_exception(logger_main, 'Error parsing conf:')
        sys.exit()

    if not args.classes:  # If we don't know where the vehicle_classes.conf is, we construct the path using defaults.
        try:
            e_conf = ConfigParser()
            e_conf.read('D:/siwim_mkiii/siwim_e/conf/siwim.conf')
            args.classes = os.path.join('D:/siwim_mkiii/sites', e_conf['global']['site'], 'conf/vehicle_classes.conf')
            SITE_DIR = os.path.join(SITE_DIR, e_conf['global']['site'])
            del e_conf
        except:
            log_exception(logger_main, 'Failed to find site name in siwim.conf. You may pass path to \'vehicle_classes.conf\' manually via -v to avoid this error.')
            sys.exit()

    # Read the vehicle_classes.conf
    try:
        veh_conf = ConfigParser()
        if not veh_conf.read(args.classes):
            log(logger_main, f'No vehicle_classes.conf found at {args.classes}!', sev=CRITICAL)
            raise SystemExit
    except SystemExit:
        sys.exit()
    except:
        log_exception(logger_main, f'Reading vehicle_classes.conf at path {args.classes} failed.')
        sys.exit()

    # Get photo locations
    PHOTOS_DICT = helpers.get_cameras_from_i_conf(conf_path=args.conf_i)

    defaults_dict = {
        'type': 'weighing',
        'origin': {
            'uuid': ID,
            'name': f'WIM {SITE_ID}',
            'address': ADDRESS,
            'serial': SERIAL,
            'location': {
                'latitude': LATITUDE,
                'longitude': LONGITUDE
            },
            'type': {
                't_install': 0,
                't_measure': [0]
            }
        }
    }

    os.makedirs(DEBUG_DIR, exist_ok=True)  # Directory where any larger debug output gets saved.
    os.makedirs(FAILED_DIR, exist_ok=True)
    os.makedirs(ERROR_DIR, exist_ok=True)
    if args.debug <= DEBUG:  # This is only relevant if debuging.
        os.makedirs(SUCCEEDED_DIR, exist_ok=True)
    if args.local:  # This is only relevant if processing a local file.
        os.makedirs(DEBUG_PACKAGE_DIR, exist_ok=True)

    if args.restart:
        log(logger_main, f'Automatic restart detected (the application likely ran out of available memory)!', sev=WARNING)
    log(logger_main, f'{APP_NAME} started with ID {ID}.')
    log(logger_main, f'Data is being sent to {server_url} with headers {server_headers}.')
    log(logger_main, f'Using cypher file {helpers.cypher_file}.')
    log(logger_main, f'Output lanes are calculated by adding {dict(conf["General"]).get("lane_offset", -1)} to SiWiM lane number.')
    log(logger_main, f'Cameras dictionary: {PHOTOS_DICT}.', sev=DEBUG)
    log(logger_main, f'Application parameters: {sys.argv[1:]}')

    # Limit memory
    if args.memory > 0:
        configure_all(args.memory, app_name=APP_NAME, logger_name='main')
        try:
            bytearray(args.memory * 1024 * 1024)
        except MemoryError:
            log(logger_main, f'Memory is limited to {args.memory} MB.')
        else:
            log(logger_main, 'Failed to limit available memory!', sev=WARNING)
    else:
        log(logger_main, 'Memory limit disabled.')

    # Set up signing stuff.
    pIface = None
    try:
        log(logger_main, 'Enabling signing interface ...')
        from EUSignCP import EUGetInterface, EULoad, EUUnload

        pIface, pInfo = setup_signing_interface(conf['General']['pwd'].encode() if 'pwd' in conf['General'] else b'siwim323')
        if not pIface:
            log(logger_main, 'Could not set up signing interface due to an error (see above or in signing log) communicating with the USB. Proceeding without signing.', sev=CRITICAL)
            encrypt_cert = bytes()
        else:
            DO_SIGN = True
            sig_dir = os.path.join(CERT_DIR, 'signature')
            if os.path.exists(sig_dir):  # If certificate directory exists, try to find the certificate
                if len(os.listdir(sig_dir)) != 1:
                    log(logger_main, f'There should be exactly one file (digital signature) in {sig_dir}! Disabling signing.', sev=CRITICAL)
                    DO_SIGN = False
                else:
                    with open(os.listdir(sig_dir)[0], 'rb') as cert:
                        encrypt_cert = cert.read()
            else:
                cert = []
                pIface.GetCertificate(pInfo["pszIssuer"], pInfo["pszSerial"], None, cert)
                encrypt_cert = cert[0]
    except Exception as e:
        log(logger_main, f'Signing not enabled due to error: {e}', sev=CRITICAL)

    if args.local:
        with open(args.local, 'r') as fd:
            msg = fd.read()
        start = time.time()
        p = Thread(target=send_files, args=(msg, dict(conf), dict(veh_conf), defaults_dict, args.debug, True))
        p.start()
        print(f'It took {round(time.time() - start, 4)} seconds to create process.')
        p.join()
        print(f'It took {round(time.time() - start, 4)} seconds for process to finish.')
        sys.exit()

    # Resend missed files
    unsent = next(os.walk(FAILED_DIR))[1]  # Gets all subdirectories in FAILED_DIR directory.
    if not NO_SEND and unsent:  # If we're not in no-send mode and there's unsent files in FAILED_DIR.
        log(logger_main, f'Attempting to resend {len(unsent)} packages from {FAILED_DIR}.')
        files = dict()
        try:
            for event in unsent:
                for f in os.listdir(os.path.join(FAILED_DIR, event)):
                    with open(os.path.join(FAILED_DIR, event, f), 'rb') as fd:
                        files[f] = fd.read()
                headers = server_headers.copy()
                headers['Api-Request-Datetime'] = f'{datetime.now():%Y-%m-%dT%H:%M:%S}{strftime("%z", gmtime())}'  # Datetime in format '2021-03-26T16:04:08+0100'
                headers['Api-Request-Uuid'] = str(uuid4())  # Each request must have a unique identifier.
                r = requests.post(server_url, headers=headers, files=files)
                if r.status_code == 200:
                    response = r.json()
                    if response['status'] != 0:  # If status is not 0 and error has occurred and should be noted and f saved for debugging.
                        log(logger_sending, f'Failed to resend {f}: {r.text}. Moving package to {ERROR_DIR}', sev=ERROR)
                        move(os.path.join(FAILED_DIR, event), os.path.join(ERROR_DIR, event))
                    else:
                        logger_info.info(f'Vehicle {f} resent successfully.')  # This logger isn't created with cestel_logging, so can't use the soon deprecated log method.
                        log(logger_sending, f'Server responded with: {response}')
            log(logger_main, f'Successfully resent {len(unsent)} packages.')
        except Exception as e:
            log(logger_main, f'Failed to resend packages with error: {e}.')

    start_tag = b'<swd '
    end_tag = b'</swd>'
    while True:
        log(logger_main, 'Connecting to I ...', sev=DEBUG)
        address = (args.ip, 8170)
        try:
            soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            soc.settimeout(60)
            soc.connect(address)
            logger_info.info('Connection to I established.')
            data = b''
            try:
                while True:
                    # Keep receiving data until we receive a complete message (or multiple)
                    new_data = soc.recv(1024)
                    log(logger_main, 'Receiving data ...', sev=DEBUG)
                    if len(new_data) == 0:  # If no data has been received, exit
                        break
                    data += new_data
                    if args.debug <= DEBUG:
                        with open(os.path.join(DEBUG_DIR, 'last_xml.xml'), 'wb') as fd:
                            fd.write(data)
                    while True:  # For each vehicle
                        log(logger_main, f'Message starts with {data[:100]} and ends with {data[-10:]}', sev=DEBUG)
                        if args.debug <= DEBUG:
                            with open(os.path.join(DEBUG_DIR, 'last_message.xml'), 'wb') as fd:
                                fd.write(data)
                        start_tag_index = data.find(start_tag)
                        if start_tag_index == -1:
                            log(logger_main, f'Didn\'t find start index in message. Breaking.', sev=DEBUG)
                            break
                        end_tag_index = data.find(end_tag, start_tag_index)
                        if end_tag_index == -1:
                            log(logger_main, f'Didn\'t find end index in message. Breaking.', sev=DEBUG)
                            break
                        start_op = time.time()  # Time how long the whole process of compression takes (prolly gonna have to be threaded)
                        data_block = data[start_tag_index:end_tag_index + len(end_tag)]

                        Thread(target=send_files, args=(data_block, dict(conf), dict(veh_conf), defaults_dict, args.debug)).start()

                        data = data[end_tag_index + len(end_tag):]
                    log(logger_main, 'Data processed.', sev=DEBUG)
            except socket.timeout:
                logger_info.info('Connection to I timed out. Attempting to reconnect.')
            finally:
                soc.close()
        except ConnectionRefusedError:
            log(logger_main, f'Connecting to {address} failed. Retrying ...')
        except socket.timeout:
            log(logger_main, f'Connecting to {address} timed out. Retrying ...')
        except KeyboardInterrupt:
            log(logger_main, 'Application stopped with keyboard interrupt.')
            break
        except (MemoryError, RuntimeError):  # While we're looking for memory errors, a RuntimeError will be thrown when a thread fails to start.
            os.execv(sys.executable, [sys.executable] + sys.argv + ['--restart'])  # Add -r parameter so that new instance can log that automatic restart happened.
        except:
            log_exception(logger_main, 'An unexpected error has occurred:')
        time.sleep(1)
    log(logger_main, f'{APP_NAME} stopped.')
