# Martin Konečnik, http://git.siwim.si/mkonecnik/integration-waga
# Script handles receiving vehicle information from SiWIM-I, converting it to json and sending it to specified url.
import argparse
import imghdr
import json
import os
import shutil
import socket
import sys
import time
from configparser import ConfigParser, MissingSectionHeaderError
from datetime import datetime
from logging import CRITICAL, DEBUG, ERROR, Formatter, INFO, StreamHandler, WARNING, getLogger
from threading import Thread
from typing import Any, Dict, List, Optional, Union
from uuid import uuid4
from warnings import filterwarnings

import requests
from cestel_console.helpers import catch_application_close, win_disable_quick_edit
from lxml import etree
from py_logging.cestel_logging import init_logger, log, log_exception
from pycountry import countries

sys.path.append('..')
import helpers

filterwarnings("ignore", category=DeprecationWarning)
requests.packages.urllib3.disable_warnings()


# Method constructs the image path and returns it (or None if no image is found).
def get_image_path(img_type: str, ts: str, lane: int) -> Optional[str]:
    logger_sending = getLogger('main')
    file_name = os.path.join(SITE_DIR, str(PHOTOS_DICT[f'{img_type}_{lane}']), f'{ts[:10]}-hh-mm-ss', f'{ts[:13]}-mm-ss', f'{ts}_{lane}_p00.jpg')
    log(logger_sending, f'Full image filename: {file_name}', sev=DEBUG)
    if os.path.exists(file_name):
        typ = imghdr.what(file_name)
        if typ != 'jpeg':
            log(logger_sending, f'Image {file_name} is not a valid jpeg. Detected type: {typ}.', sev=DEBUG)
            return None
        return file_name
    else:
        log(logger_sending, f'Image {file_name} not found.', sev=DEBUG)
        return None


def reclassify_comark(cls: int, id: Optional[str] = None) -> int:  # ts is needed to write event timestamp for busses that got detected.
    logger_saving = getLogger('main')
    if cls == 0 or cls == 1:  # Unidentified or Motorcycle.
        return 20
    elif cls == 2 or cls == 3:  # Car or van.
        return 3
    elif cls == 4 or cls == 5:  # Bus or coach.
        return 19
    elif cls == 6:  # Truck.
        return 4
    elif cls == 7:  # Articulated truck.
        return 8
    elif cls == 8:  # Semi-truck.
        return 12
    else:
        log(logger_saving, f'Comark ts {id} contains invalid EUR6 class: {cls}!', sev=ERROR)
        return 20


def get_image_dict(image_type: str, ts: str, lane: int, tz: str, use_image_time: bool = False, local: bool = False) -> Optional[dict]:
    send_logger = getLogger('main')
    log(send_logger, f'Looking for {image_type}', sev=DEBUG)
    image_path = get_image_path(image_type, ts, lane)
    if image_path:
        log(send_logger, f'Found {image_type} image: {image_path}', sev=DEBUG)
        if local:  # Copy the images if running with local parameter.
            shutil.copyfile(image_path, f'{os.path.join(DEBUG_PACKAGE_DIR, image_type)}.jpg')
        img_hash = list()
        try:
            if pIface:
                pIface.HashFile(image_path, None, img_hash)  # Saves the hash into img_hash
                log(getLogger('signing'), f'Hash calculated for {image_type} image {os.path.basename(image_path)} {helpers.to_hex(img_hash[0])}.')
            else:
                log(getLogger('signing'), 'Could not contact USB to calculate image hash!', sev=ERROR)
                img_hash = [b'hash']
        except RuntimeError as e:
            log(getLogger('signing'), f'An error has occurred when trying to calculate hash for {image_type} image {os.path.basename(image_path)}. Hash is {helpers.to_hex(img_hash[0])}. Error: {e}', sev=ERROR)
            return None
        return {
            'type': image_type,
            'baseurl': os.path.dirname(image_path),  # Used for sending images. Should be set to empty string once done.
            'name': os.path.basename(image_path),
            'hash': [
                'gost34311',
                helpers.to_hex(img_hash[0])
            ],
            'datetime': helpers.get_photo_ts(image_path, tz) if use_image_time else helpers.convert_ts(ts, tz)
        }


def add_photo(img_dict: dict, img_type: str, media: list, ts: str, lane: int, tz: str, local: bool = False) -> None:
    if img_dict[img_type] is None:  # If plate hasn't been found yet.
        img_dict[img_type] = get_image_dict(img_type, ts, lane, tz, local=local)  # Check if image exists.
        if img_dict[img_type]:  # If image was found, append the data.
            media.append(img_dict[img_type])


def create_measurements_key(veh: etree.Element) -> List[Dict[str, str]]:
    """ Builds the measurements list. """
    weights = [helpers.convert_kN_to_kg(el.text) for el in veh.xpath('acws/w')]
    measures: List[Dict[str, str]] = [
        {
            'key': 'speed-main',
            'value': str(helpers.convert_m_per_s_to_km_per_h(veh.find('v').text))
        },
        {
            'key': 'weight-full',
            'value': str(sum(weights))
        },
        {
            'key': 'axlecount',
            'value': veh.find('naxles').text
        },
        {
            'key': 'road-temperature',
            'value': round(float(veh.find('T').text))
        },
        {
            'key': 'air-temperature',
            'value': '-273'
        }
    ]

    if helpers.classify_ua(veh.find('cls').text) == 20:  # Unclassified vehicles should always be marked as overweight.
        measures.append(
            {
                'key': 'overweighted',
                'value': 1
            }
        )

    single_load = list()  # List of axle weights.
    wheel_type = list()  # List of wheel types (0 - undefined, 1 - single pitched, 2 - gable).
    axle_count = 1  # Index of axle/group.
    for weight in weights:
        single_load.append(
            {
                'key': f'axleload-{axle_count}',
                'value': str(weight)
            }
        )
        wheel_type.append(
            {
                'key': f'wheeltype-{axle_count}',
                'value': '0'
            }
        )
        axle_count += 1

    axle_count = 1
    axle_distances = list()  # List of axle distances.
    for d in veh.xpath('ads/d'):
        axle_distances.append(
            {
                'key': f'axledistance-{axle_count}',
                'value': str(helpers.convert_m_to_mm(d.text))
            }
        )
        axle_count += 1

    return measures + single_load + axle_distances + wheel_type  # Concatenate the lists.


def create_vehicle_key(wim: etree.Element, anpr: etree.Element) -> Dict[str, List[Union[Dict[str, str], List[str]]]]:
    veh_dict: Dict[str, List[Union[Dict[str, str], List[int]]]] = dict()
    if anpr is not None:
        veh_dict['plate'] = [
            {
                # ffgroup currently saves XX when there's no country code.
                'country': int(countries.get(alpha_3=anpr.find('country').text).numeric) if anpr.find('country') is not None and anpr.find('country').text is not None and anpr.find('country').text != 'XX' else 0,
                'text': anpr.find('lp').text if anpr.find('lp') is not None else str(),
                'placement': anpr.find('location').text if anpr.find('location') is not None else 'front',
                'precision': round(float(anpr.find('confidence').text) * 100) if anpr.find('confidence') is not None else 0
            }
        ]
    else:
        veh_dict['plate'] = [
            {
                'country': 0,
                'text': 'notrecognized',
                'placement': 'front',
                'precision': 0
            }
        ]
    veh_dict['params'] = [
        {
            'key': 'class',
            'value': str(helpers.classify_ua(wim.find('cls').text))
        },
        {
            'key': 'lane',
            'value': conf_dict['General']['lanes'].split(',')[int(wim.find('lane').text) - 1]
        }
    ]
    # Add axle groups in expected format.
    axle_groups = wim.find('axgrps').text
    axle_index = 1
    veh_dict['axlesLayout'] = list()  # Axle groups in UA format.
    for group in axle_groups:  # Iterate over axgrps string.
        ua_group = list()
        for i in range(int(group)):  # For each number in string add as many axles.
            ua_group.append(axle_index)
            axle_index += 1
        veh_dict['axlesLayout'].append(ua_group)
    return veh_dict


def process_message(msg: Union[str, etree.Element], conf: dict, veh_conf: dict) -> Optional[dict]:
    logger = getLogger('main')
    try:
        xml = etree.fromstring(msg)
        event = xml.xpath('site/vehicles/vehicle')[0]
    except ValueError:  # If we get value error, just assume it's already an etree.Element. This is used by --daily operation.
        event = msg
    except etree.XMLSyntaxError:
        log(logger, f'Failed to parse string {msg}')
        return None

    veh: etree.Element = event.find('wim')
    comark: etree.Element = event.find('lwh')

    json_dict: Dict[str, Any] = defaults_dict.copy()  # We don't want to edit the defaults dictionary
    json_dict['uuid'] = str(uuid4())  # Generate event uuid.
    try:
        # Only use our data, if vehicle is classified as 4+ on eur6 and disregard flag isn't set.
        if veh is not None and helpers.classify_eur6(veh.find('cls').text) >= 4 and not helpers.check_flag(veh.find('flags').text, 1, 2):
            flags = veh.find('flags').text
            siwim_ts = veh.find('ts').text

            global DEFAULT_TEMP
            DEFAULT_TEMP = round(float(veh.find('T').text))

            # Add measurements.
            json_dict['measurements'] = create_measurements_key(veh)

            if helpers.check_flag(flags, 0, 8):  # Blur.
                json_dict['flags'] = [{
                    'key': 'INVALID_MEASUREMENT',
                    'value': 'Measurement likely inaccurate'
                }]

            # Add vehicle info.
            json_dict['vehicle'] = create_vehicle_key(veh, event.find('anpr'))

            # Add datetime info.
            timezone = int(veh.find('tzbias').text) / 3600
            if not timezone.is_integer():
                log(logger, f'Adding timezone information to string is implemented only for integer values. Timezone {timezone} is not supported.', sev=CRITICAL)
            timezone_str = f'0{int(timezone)}:00' if timezone < 10 else f'{timezone}:00'
            json_dict['datetime'] = helpers.convert_ts(siwim_ts, timezone_str)  # Convert to their format.
            global DEFAULT_TZ
            DEFAULT_TZ = timezone_str

            # Add photos info.
            img_dict = {
                'plate': None,
                'front': None,
                'side': None
            }
            json_dict['media'] = list()
            start = time.time()
            while time.time() - start < args.timeout:  # We need hashes, so we need to wait for photos even if we're sending them later.
                add_photo(img_dict, 'plate', json_dict['media'], siwim_ts, int(veh.find('lane').text), timezone_str, local=args.local)
                add_photo(img_dict, 'front', json_dict['media'], siwim_ts, int(veh.find('lane').text), timezone_str, local=args.local)
                add_photo(img_dict, 'side', json_dict['media'], siwim_ts, int(veh.find('lane').text), timezone_str, local=args.local)
                log(logger, f'Found {len(json_dict["media"])} images.', sev=DEBUG)
                if len(json_dict['media']) > 3:
                    log(logger, f'Too many photos: {json_dict["media"]}.', sev=ERROR)
                elif len(json_dict['media']) == 3:
                    log(logger, f'Found all images.', sev=DEBUG)
                    break
                time.sleep(1)
            helpers.are_images_present(img_dict, siwim_ts)  # Checks if we found all images and prints warnings if that's not the case.

        elif comark is not None:  # If there's no wim tag, or we chose to ignore it, but comark exists.
            if DEFAULT_TZ is None:  # This can only happen at application start if a comark vehicle arrives before a wim vehicle and is thus unimportant.
                return None

            json_dict['vehicle']['params'] = [
                {
                    'key': 'class',
                    'value': str(reclassify_comark(int(float(comark.find('cls').text)), id=comark.find('ts').text))
                },
                {
                    'key': 'lane',
                    'value': conf_dict['General']['lanes'].split(',')[int(comark.find('lane').text) - 1]
                }
            ]

            json_dict['measurements'] = [
                {
                    'key': 'speed-main',
                    'value': str(helpers.convert_m_per_s_to_km_per_h(comark.find('v').text))
                },
                {
                    'key': 'road-temperature',
                    'value': DEFAULT_TEMP
                }
            ]

            json_dict['datetime'] = helpers.convert_ts(comark.find('ts').text, DEFAULT_TZ)

        else:
            # This can only execute if wim tag was present and ignored, but lwh was not.
            log(logger, f'{veh.find("ts").text} with flags {veh.find("flags").text} and eur6 class {helpers.classify_eur6(veh.find("cls").text)} discarded. No comark data found.')
            return None
    except:
        json_dict['flags'] = [{
            'key': 'INVALID_MEASUREMENT',
            'value': 'An unexpected error has occurred'
        }]
        log_exception(logger, 'An exception has occurred while parsing xml:')
        with open(os.path.join(DEBUG_DIR, f'{veh.find("ts").text}.xml'), 'w') as d:  # Save the message that caused exception.
            d.write(msg)
    return json_dict


def dump_package(directory: str, files: dict, exist_ok: bool = False, headers: dict = None, error: str = str()) -> None:
    """ Saves the package to disk
    :param directory: Directory that we want to save to.
    :param files: Dictionary of tuples containing file name and file data.
    :param exist_ok: Passed to os.makedirs.
    :param headers: If passed, headers are written to info.txt.
    :param error: Error message that was received.
    """
    os.makedirs(directory, exist_ok=exist_ok)
    if headers:
        with open(os.path.join(directory, 'info.txt'), 'w') as f:
            f.write(f'{headers}\n')
            f.write(error)
    for fname, fdata in files.items():
        with open(os.path.join(directory, fname), 'wb') as f:
            f.write(fdata[1])


def send_files(msg: str, conf: dict, veh_classes: dict) -> None:
    # The logger must be initialised in the thread it's logging in or style will be lost (naturally it can be done otherwise, but not worth it here).
    sending_logger = getLogger('sending')
    console_logger = getLogger('info')
    json_dict = process_message(msg, conf, veh_classes)  # We have to pass a copy of the defaults dictionary, or we'll just be adding to it constantly.
    if json_dict:
        ts = datetime.strptime(json_dict['datetime'][:-6], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
        headers = server_headers.copy()  # Make a copy of the default dict.
        headers['Api-Request-Datetime'] = f'{datetime.now():%Y-%m-%dT%H:%M:%S}{time.strftime("%z", time.gmtime())}'  # Datetime in format '2021-03-26T16:04:08+0100'

        files = dict()
        for image in json_dict['media']:
            typ = image['type']
            name = image['name']
            try:
                files[f'{typ}-{name}'] = (f'{typ}-{name}', open(os.path.join(image['baseurl'], name), 'rb').read(), 'application/octet-stream')
            except FileNotFoundError:
                log(sending_logger, f'Photo {image["baseurl"]} not found!', sev=ERROR)

        for i in range(len(json_dict['media'])):  # Set all baseurls to empty string and fix names.
            json_dict['media'][i]['baseurl'] = str()
            json_dict['media'][i]['name'] = f'{json_dict["media"][i]["type"]}-{json_dict["media"][i]["name"]}'  # Change the name variable, to reflect the key value.

        if DO_SIGN:
            signature = sign_data(json.dumps(json_dict, ensure_ascii=False).encode())

            if signature is None:
                log(sending_logger, f'Signing failed! See signing log for more info. Dumping {ts} to {FAILED_DIR}.', sev=ERROR)
                with open(os.path.join(FAILED_DIR, f'{ts}.json'), 'w') as f:
                    json.dump(json_dict, f)
                return
        else:
            signature = bytes()

        if encrypt_cert:
            files[f'{ts}.crt'] = (f'{ts}.crt', encrypt_cert, 'application/octet-stream')
        else:
            files[f'{ts}.crt'] = (f'{ts}.crt', bytes(), 'application/octet-stream')
        if signature:
            files[f'{ts}.sign'] = (f'{ts}.sign', signature, 'application/octet-stream')
        else:
            files[f'{ts}.sign'] = (f'{ts}.sign', bytes(), 'application/octet-stream')
        files[f'{ts}.json'] = (f'{ts}.json', json.dumps(json_dict, ensure_ascii=False).encode(), 'application/json')  # Add json last, since baseurl has to be cleared first.

        if NO_SEND or args.debug <= DEBUG:
            dump_package(os.path.join(DEBUG_PACKAGE_DIR, ts), files, exist_ok=True, headers=headers)

        if NO_SEND:  # Don't send if running test.
            log(sending_logger, 'Not sending anything, because NO_SEND is set. This gets set automatically if running with --local or --ip parameters.', sev=CRITICAL)
            return

        retry = 0
        while retry < MAX_RETRIES:
            try:
                headers['Api-Request-Uuid'] = str(uuid4())  # Each request must have a unique identifier.
                r = requests.post(server_url, headers=headers, files=files, timeout=2, verify=False)
            except:
                console_logger.warning(f'Server did not respond for {ts}. Retries left {MAX_RETRIES - retry - 1}')
                retry += 1
                time.sleep(1)
                continue
            if r.ok:
                response = r.json()
                if response['status'] != 'Accepted':  # If status is not 0 and error has occurred and should be noted and event saved for debugging.
                    log(sending_logger, f'Failed to send {ts}: {r.text}. Error: {response.get("error", "none")}. Saving package to {ERROR_DIR}', sev=ERROR)
                    dump_package(os.path.join(ERROR_DIR, ts), files, exist_ok=True, headers=headers, error=r.text)  # If we get a non-zero response, something is wrong so we dump the files.
                else:
                    console_logger.info(f'{ts} sent successfully.')  # This logger isn't created with cestel_logging, so can't use the soon deprecated log method.
                    log(sending_logger, f'Server responded with: {response}.', sev=DEBUG)
                    if args.debug <= DEBUG:
                        with open(os.path.join(SUCCEEDED_DIR, ts + '.json'), 'w') as outfile:
                            json.dump(json_dict, outfile)
                break
            elif 400 <= r.status_code < 500:
                log(sending_logger, f'Client error: {r.text}. Authentication headers: {headers}.', sev=ERROR)
                dump_package(os.path.join(ERROR_DIR, ts), files, exist_ok=True, headers=headers, error=r.text)
                break
            elif 500 <= r.status_code < 600:
                log(sending_logger, f'Couldn\'t reach server: {r.text}.', sev=WARNING)
                if retry == MAX_RETRIES - 1:  # We only want to save it, if we're not retrying anymore.
                    dump_package(os.path.join(FAILED_DIR, ts), files, exist_ok=True, headers=headers, error=r.text)
            else:
                log(sending_logger, f'Sending attempt failed with status code {r.status_code} and response {r.text}.', sev=DEBUG)
            retry += 1
            time.sleep(1)
        else:
            log(sending_logger, f'Failed to send vehicle {ts} after {retry} attempts!', sev=WARNING)
            dump_package(os.path.join(FAILED_DIR, ts), files, exist_ok=True)
        log(sending_logger, f'Sending process for {json_dict["datetime"]} exited.')


def resend_vehicle(event: str, files: Optional[Dict[str, tuple]] = None) -> None:
    if not files:  # Files are passed only for events where signing failed and thus files need to be generated again.
        files = dict()
        for f in os.listdir(os.path.join(FAILED_DIR, event)):
            with open(os.path.join(FAILED_DIR, event, f), 'rb') as fd:
                files[f] = fd.read()
    headers = server_headers.copy()
    headers['Api-Request-Datetime'] = f'{datetime.now():%Y-%m-%dT%H:%M:%S}{time.strftime("%z", time.gmtime())}'  # Datetime in format '2021-03-26T16:04:08+0100'
    headers['Api-Request-Uuid'] = str(uuid4())  # Each request must have a unique identifier.
    try:
        r = requests.post(server_url, headers=headers, files=files, timeout=2, verify=False)
    except ConnectionError as e:
        log(getLogger('sending'), f'Failed to resend {event}. Error: {e}', sev=ERROR)
        return
    if r.status_code == 200:
        response = r.json()
        if response['status'] != 'Accepted':  # If status is not 0 and error has occurred and should be noted and f saved for debugging.
            log(getLogger('sending'), f'Failed to resend {event}: {r.text}. Error: {response.get("error", None)} Moving package to {ERROR_DIR}', sev=ERROR)
            shutil.move(os.path.join(FAILED_DIR, event), os.path.join(ERROR_DIR, event))
        else:
            log(getLogger('sending'), f'{event} resent successfully.')  # This logger isn't created with cestel_logging, so can't use the soon deprecated log method.
            log(getLogger('sending'), f'Server responded with: {response}.', sev=DEBUG)
    elif 500 <= r.status_code < 600:
        log(getLogger('sending'), f'Couldn\'t reach server: {r.text}.', sev=WARNING)
    else:
        log(getLogger('sending'), f'Sending attempt failed with status code {r.status_code} and response {r.text}.')
        shutil.move(os.path.join(FAILED_DIR, event), os.path.join(ERROR_DIR, event))


def setup_signing_interface(password: bytes) -> tuple:  # Returns tuple of EUGetInterface and info
    start = time.time()
    logger_sign = getLogger('signing')
    EULoad()
    log(logger_sign, f'Load time: {round(time.time() - start, 4)}s', sev=DEBUG)
    pIface = EUGetInterface()
    try:
        pIface.SetUIMode(False)
        pIface.Initialize()
        pIface.SetUIMode(False)
    except Exception as e:
        log(logger_sign, f'Initialize failed: {e}', sev=ERROR)
        EUUnload()
        return None, None

    log(logger_sign, 'Library Initialized', sev=DEBUG)

    dwType = 0
    lDescription = []
    try:
        pIface.EnumKeyMediaTypes(dwType, lDescription)
        while lDescription[0] != 'е.ключ ІІТ Алмаз-1К':
            dwType += 1
            if not pIface.EnumKeyMediaTypes(dwType, lDescription):
                log(logger_sign, 'KeyMedia not found', sev=ERROR)
                pIface.Finalize()
                EUUnload()
                return None, None

    except Exception as e:
        log(logger_sign, f'EnumKeyMediaTypes failed: {e}', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None

    lDescription = []
    if not pIface.EnumKeyMediaDevices(dwType, 0, lDescription):
        log(logger_sign, 'Device not found', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None
    log(logger_sign, f'Device description: {lDescription[0]}', sev=DEBUG)

    pKM = {'szPassword': password, 'dwDevIndex': 0, 'dwTypeIndex': dwType}
    try:
        pInfo = dict()  # Info about certificate is stored here.
        pIface.ReadPrivateKey(pKM, pInfo)
    except Exception as e:
        log(logger_sign, f'Key reading failed: {e}', sev=ERROR)
        pIface.Finalize()
        EUUnload()
        return None, None

    log(logger_sign, 'Key read successfully.', sev=DEBUG)
    return pIface, pInfo


def sign_data(def0_file: bytes) -> Optional[bytes]:
    signed = list()
    try:
        pIface.SignDataInternal(True, def0_file, len(def0_file), None, signed)
    except Exception as e:
        log(getLogger('signing'), f'SignData failed: {e}', sev=ERROR)
        return None
    return signed[0]


if __name__ == '__main__':
    APP_NAME = 'Converter for WAGA 1.0.5b v1.0.1'
    DEBUG_DIR = 'debug'
    DEBUG_PACKAGE_DIR = 'package'
    FAILED_DIR = 'failed_to_send'
    ERROR_DIR = 'error'  # Directory to which files that failed signing are saved.
    SUCCEEDED_DIR = 'succeeded_sending'
    CERT_DIR = '../certificate'
    MAX_RETRIES = 5
    server_url = 'https://waga-api.ukravtodor.gov.ua/api/wim/events'
    server_headers = {
        'Authorization': 'Token ZWMyMTA2MTYtY2EzZi00NTMyLWJmZjctNWQwOTBmYzRmMjAw'
    }
    DEFAULT_TEMP: Optional[int] = None  # Holds the value of last temperature, so we can send it when there's no temperature for reasons.
    DEFAULT_TZ: Optional[str] = None  # Holds the value of last bias for same reason as temp.

    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--conf', help='path to the configuration file', default=r'..\ua.conf', type=str)
    parser.add_argument('--conf_i', help='path to SiWIM-I configuration file', default=r'D:\siwim_mkiii\siwim_i\conf\conf.xml', type=str)
    parser.add_argument('--daily', help='convert a daily xml file; optionally, next two numeric values are read as start and end time', nargs='+', default=None)
    parser.add_argument('--disable_signing', help='disables signing functionality', action='store_true')
    parser.add_argument('-d', '--debug', help='run application with debug output', nargs='?', const=DEBUG, default=WARNING, type=int)
    parser.add_argument('--export_class', help='passing this will save classification information in excel friendly format', action='store_true')
    parser.add_argument('-i', '--ip', help=f'IP of I to which we\'re connecting', default='127.0.0.1', type=str)
    parser.add_argument('-l', '--local', help='define local xml to process', type=str)
    parser.add_argument('-p', '--port', help='to which I port we want to connect', default=8175, type=int)
    parser.add_argument('--siwim_e_conf', help='pass to overwrite siwim_e conf location', default=r'D:\siwim_mkiii\siwim_e\conf\siwim.conf', type=str)
    parser.add_argument('-t', '--timeout', help=f'how long to wait for images before deciding they don\'t exist', default=10, type=int)
    parser.add_argument('-v', '--classes', help=f'path to vehicle_classes.conf, if not passed it will be determined using E\'s conf file', type=str)
    args = parser.parse_args()

    if args.local or parser.get_default('ip') != args.ip:  # Disable sending if we're processing a local file or receiving data from a non-localhost IP.
        NO_SEND = True
        SITE_DIR = r'D:\Projects\Python\integration_ukraine\xmls\photos'  # If running locally, the whole image path has to be passed.
    else:
        NO_SEND = False
        SITE_DIR = r'D:\siwim_mkiii\sites'  # When running on site, partial path will be deduced based on E's conf.

    # Initialise the application's loggers.
    logger_main = init_logger('main', to_console=True, console_level=INFO if args.debug > INFO else args.debug, level=INFO if args.debug > INFO else args.debug)
    init_logger('sending', to_console=True, console_level=args.debug, level=args.debug)
    init_logger('signing', level=args.debug, to_console=True, console_level=args.debug)
    # Logger that's logging to console only isn't implemented in py_logging 1.0 yet.
    logger_info = getLogger('info')
    console = StreamHandler()
    console.setLevel(INFO)
    console.setFormatter(Formatter(fmt='%(name)-9s %(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
    logger_info.addHandler(console)
    logger_info.setLevel(INFO)

    try:
        # And the rest from ua.conf.
        conf = ConfigParser()
        conf.read(args.conf, encoding='utf-8-sig')

        defaults_dict = {
            'deviceId': conf['General']['waga_uuid'],
            'eventDeviceId': int(conf['General']['site_id']),
            'origin': {
                'uuid': conf['General']['waga_uuid'],
                'name': conf['General']['waga_name'],
                'address': conf['General']['address'],
                'serial': conf['General']['serial'],
                'location': {
                    'latitude': conf['General']['latitude'],
                    'longitude': conf['General']['longitude']
                },
                'type': {
                    't_install': 1  # High-speed WIM.
                }
            },
            'measurements': list(),
            'vehicle': {
                'plate': list(),
                'params': list()
            },
            'media': list()
        }

        conf_dict = dict(conf)  # Imperfect conversion, but good enough for what we're doing.
        conf_dict['General']['lanes'] = conf['General']['lanes']
    except MissingSectionHeaderError:
        log(logger_main, 'Failed to find section headers in conf. It is likely that it is not saved with proper encoding. Make sure ua.conf is saved with UTF-8 encoding and not UTF8 Signature encoding.', sev=CRITICAL)
        sys.exit()
    except:
        log_exception(logger_main, 'Error parsing conf:')
        sys.exit()

    if not args.classes:  # If we don't know where the vehicle_classes.conf is, we construct the path using defaults.
        try:
            # We get site name from SiWIM-E conf.
            e_conf = ConfigParser()
            e_conf.read(args.siwim_e_conf)
            args.classes = os.path.join('D:/siwim_mkiii/sites', e_conf['global']['site'], 'conf/vehicle_classes.conf')
            SITE_DIR = os.path.join(SITE_DIR, e_conf['global']['site'])
            del e_conf  # We don't need to keep E's conf around anymore.
        except:
            log_exception(logger_main, 'Failed to find site name in siwim.conf. You may pass path to \'vehicle_classes.conf\' manually via -v to avoid this error.')
            sys.exit()

    # Read the vehicle_classes.conf
    try:
        veh_conf = ConfigParser()
        if not veh_conf.read(args.classes):
            log(logger_main, f'No vehicle_classes.conf found at {args.classes}!', sev=CRITICAL)
            raise SystemExit
        veh_conf = dict(veh_conf)
    except SystemExit:
        sys.exit()
    except:
        log_exception(logger_main, f'Reading vehicle_classes.conf at {args.classes} failed.')
        sys.exit()

    PHOTOS_DICT = helpers.get_cameras_from_i_conf(conf_path=args.conf_i)

    os.makedirs(FAILED_DIR, exist_ok=True)
    os.makedirs(ERROR_DIR, exist_ok=True)  # Directory where vehicles that failed sending due to an error with file or signing are saved.
    os.makedirs(DEBUG_DIR, exist_ok=True)  # Directory where any larger debug output gets saved.
    if args.debug <= DEBUG:
        os.makedirs(SUCCEEDED_DIR, exist_ok=True)
    if args.local:  # This is only relevant if processing a local file.
        os.makedirs(DEBUG_PACKAGE_DIR, exist_ok=True)

    if args.export_class:
        args.export_class = os.path.join('..', f'classification_WIM-{str(defaults_dict["eventDeviceId"]).rjust(3, "0")}_{os.path.basename(args.daily[0]).split(".")[0]}.txt')

    log(logger_main, f'{APP_NAME} started with ID {defaults_dict["eventDeviceId"]}. Listening on port {args.port}.')
    log(logger_main, f'Data is being sent to {server_url} with headers {server_headers}.')
    log(logger_main, f'Using cypher file {helpers.cypher_file}.')
    log(logger_main, f'Output lanes are mapped to {conf_dict["General"]["lanes"]}.')
    log(logger_main, f'Cameras dictionary: {PHOTOS_DICT}.', sev=DEBUG)
    log(logger_main, f'Application parameters: {sys.argv[1:]}')

    win_disable_quick_edit(logger_name='main')
    catch_application_close(app_name=APP_NAME, logger_name='main')

    # Set up signing stuff.
    pIface = None
    encrypt_cert = bytes()
    try:
        log(logger_main, 'Enabling signing interface ...')
        from EUSignCP import EUGetInterface, EULoad, EUUnload

        pIface, pInfo = setup_signing_interface(conf['General']['pwd'].encode() if 'pwd' in conf['General'] else b'siwim323')
        if not pIface:
            log(logger_main, 'Could not set up signing interface due to an error (see above or in signing log) communicating with the USB. Proceeding without signing.', sev=CRITICAL)
            DO_SIGN = False
            raise Exception('Can not communicate with signing device.')
        else:
            if args.disable_signing:  # Even if signing is disabled we still need the USB device in order to calculate hashes.
                DO_SIGN = False
                raise Exception('Disabled')
            else:
                DO_SIGN = True
                sig_dir = os.path.join(CERT_DIR, 'signature')
                if os.path.exists(sig_dir):  # If certificate directory exists, try to find the certificate
                    if len(os.listdir(sig_dir)) != 1:
                        log(logger_main, f'There should be exactly one file (digital signature) in {sig_dir}! Disabling signing.', sev=CRITICAL)
                        DO_SIGN = False
                    else:
                        with open(os.listdir(sig_dir)[0], 'rb') as cert:
                            encrypt_cert = cert.read()
                else:
                    cert = []
                    pIface.GetCertificate(pInfo["pszIssuer"], pInfo["pszSerial"], None, cert)
                    encrypt_cert = cert[0]
    except Exception as e:
        log(logger_main, f'Signing not enabled due to error: {e}.', sev=CRITICAL)
        DO_SIGN = False

    if args.local:
        with open(args.local, 'r') as fd:
            msg = fd.read()
        start = time.time()
        p = Thread(target=send_files, args=(msg, conf, veh_conf))
        p.start()
        print(f'It took {round(time.time() - start, 4)} seconds to create process.')
        p.join()
        print(f'It took {round(time.time() - start, 4)} seconds for process to finish.')
        sys.exit()

    # Resend signed files.
    unsent = next(os.walk(FAILED_DIR))[1]  # Gets all subdirectories in FAILED_DIR directory.
    if not NO_SEND and unsent:  # If we're not in no-send mode and there's unsent files in FAILED_DIR.
        log(logger_main, f'Attempting to resend {len(unsent)} packages from {FAILED_DIR}. This may take a while (up to 0.5 second per event) ...')
        try:
            for event in unsent:
                resend_vehicle(event)
            log(logger_main, f'Successfully resent {len(unsent)} packages.')
        except Exception as e:
            log(logger_main, f'Failed to resend packages with error: {e}.', sev=WARNING, exc_info=True)

    # Sign and resend unsigned files.
    unsent = [f for f in os.listdir(FAILED_DIR) if os.path.isfile(f)]
    if not NO_SEND and unsent:
        log(logger_main, f'Attempting to sign and resend {len(unsent)} files from {FAILED_DIR}.')
        try:
            for event in unsent:
                with open(os.path.join(FAILED_DIR, event), 'r') as fd:
                    json_dict = json.load(fd)

                if DO_SIGN:
                    signature = sign_data(json.dumps(json_dict, ensure_ascii=False).encode())

                    if signature is None:
                        log(logger_main, f'Signing failed! See signing log for more info. Dumping {ts} to {FAILED_DIR}.', sev=ERROR)
                        with open(os.path.join(FAILED_DIR, f'{ts}.json'), 'w') as f:
                            json.dump(json_dict, f)
                        break
                else:
                    signature = bytes()

                files = dict()
                ts = datetime.strptime(json_dict['datetime'], '%Y-%m-%dT%H:%M:%S.%f%z').strftime('%Y-%m-%d-%H-%M-%S-%f')

                if encrypt_cert:
                    files[f'{ts}.crt'] = (f'{ts}.crt', encrypt_cert, 'application/octet-stream')
                else:
                    files[f'{ts}.crt'] = (f'{ts}.crt', bytes(), 'application/octet-stream')
                if signature:
                    files[f'{ts}.sign'] = (f'{ts}.sign', signature, 'application/octet-stream')
                else:
                    files[f'{ts}.sign'] = (f'{ts}.sign', bytes(), 'application/octet-stream')
                files[f'{ts}.json'] = (f'{ts}.json', json.dumps(json_dict, ensure_ascii=False).encode(), 'application/json')  # Add json last, since baseurl has to be cleared first.

                resend_vehicle(event, files=files)
        except Exception as e:
            log(logger_main, f'Failed to resend json files with error: {e}.', exc_info=True)

    shutil.rmtree(FAILED_DIR)  # After sending is done, remove whatever is left.
    os.makedirs(FAILED_DIR)  # Recreate the directory.

    start_tag = b'<swd '
    end_tag = b'</swd>'
    while True:
        log(logger_main, 'Connecting to I ...', sev=DEBUG)
        address = (args.ip, args.port)
        try:
            soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            soc.connect(address)
            soc.settimeout(60)
            logger_info.info('Connection to I established.')
            data = b''
            try:
                while True:
                    # Keep receiving data until we receive a complete message (or multiple)
                    new_data = soc.recv(1024)
                    if len(new_data) == 0:  # If no data has been received, exit
                        break
                    data += new_data
                    if args.debug <= DEBUG:
                        with open(os.path.join(DEBUG_DIR, 'last_xml.xml'), 'wb') as fd:
                            fd.write(data)
                    while True:  # For each vehicle
                        start_tag_index = data.find(start_tag)
                        if start_tag_index == -1:
                            break
                        end_tag_index = data.find(end_tag, start_tag_index)
                        if end_tag_index == -1:
                            break
                        data_block = data[start_tag_index:end_tag_index + len(end_tag)]

                        try:
                            Thread(target=send_files, args=(data_block, conf_dict, veh_conf)).start()
                        except RuntimeError as e:
                            log(logger_main, f'{e}. This means that there is a problem with threads not closing. It is possible that this resolves itself, but report it to application maintainer regardless.', sev=CRITICAL)
                            continue

                        data = data[end_tag_index + len(end_tag):]
            except ConnectionResetError:
                log(logger_main, 'Connection to I lost. Attempting to reconnect.', sev=WARNING)
            except socket.timeout:
                logger_info.info('Connection to I timed out. Attempting to reconnect.')
            finally:
                soc.close()
        except ConnectionRefusedError:
            log(logger_main, f'Connecting to {address} failed. Retrying ...')
        except socket.timeout:
            log(logger_main, f'Connection to {address} timed out. Retrying ...')
        except KeyboardInterrupt:
            log(logger_main, 'Application stopped with keyboard interrupt.')
            break
        except:
            log_exception(logger_main, 'An unexpected error has occurred:')
        time.sleep(1)
    EUUnload()
    log(logger_main, f'{APP_NAME} stopped.')
