Flatten create structure

Signed-off-by: Ralf Zerres <ralf.zerres@networkx.de>

advotracker_client/examples/csv-test/.env

@@ -0,0 +1,7 @@
+# set the language variable used in test-runs
+#test lang=de
+TEST_LANG=en
+#RUST_LOG=csv-test=trace
+RUST_LOG=trace
+USERNAME=ralf
+DB_DRIVER=sqlite

advotracker_client/examples/csv-test/data/POLLFNR_MINI.txt

@@ -0,0 +1,23 @@
+DION VERS POLLFNR
+1 AS 1
+1 AS 11
+1 AS 123456789
+9 AS 99
+9 AS 999
+1 AS 1511111111
+1 AS 1511111119
+1 AS 9000000001
+1 AS 9000000002
+1 AS 9999999991
+1 AS 9999999992
+1 AS 9999999999
+3 AS 1515735920
+3 AS 9999999991
+3 AS 9999999992
+3 AS 9999999999
+9 AS 9927764110
+9 AS 9927764251
+9 AS 9927764426
+9 AS 9999999991
+9 AS 9999999992
+9 AS 9999999999

advotracker_client/examples/csv-test/data/POLLFNR_WOECHENTLICH.txt

@@ -0,0 +1 @@
+../../../data/POLLFNR_WOECHENTLICH.txt

advotracker_client/examples/csv-test/main.rs

@@ -0,0 +1,409 @@
+/*
+ * advotracker - Hotline tackingtool for Advocats
+ *
+ * Copyright 2020 Ralf Zerres <ralf.zerres@networkx.de>
+ * SPDX-License-Identifier: (0BSD or MIT)
+ */
+
+use chrono::{Local, DateTime};
+use locales::t;
+//use serde::{Deserialize, Serialize};
+use serde::Deserialize;
+use std::{
+    collections::HashMap,
+    env,
+    {error::Error, process},
+    //path::{Path, PathBuf},
+};
+use tracing::{debug, trace, Level};
+
+use advotracker::data::structures::{PolicyCode, PolicyList, PolicyDataList, PolicyData};
+
+// include modules
+mod parse_args;
+
+/// respect environment variables set in .env files
+/// located in the current call directory
+/// this is primarily used in testing scenarios (eg. debugging)
+#[derive(Debug, Deserialize)]
+struct Environment {
+    test_lang: String,
+    log: String,
+}
+
+/// export as csv format
+/// https://docs.rs/csv/1.1.3/csv/cookbook/index.html
+/// https://blog.burntsushi.net/csv/
+fn export(p: &mut String, lang: &String) -> Result<u64, Box<dyn Error>> {
+    use std::fs::File;
+    use std::path::Path;
+    //use std::ffi::OsStr;
+    use std::io::prelude::*;
+
+    let mut res = t!("csv.export.started", lang);
+    let mut state = t!("state.started", lang);
+    let dt_start: DateTime<Local> = Local::now();
+
+    trace!(target: "csv-test",
+           process = ?res,
+           state = ?state,
+           date_start = ?dt_start.to_string());
+
+    // Note: slash syntax also works on Windows!
+    let path = Path::new(p);
+
+    // open the file descriptor
+    let mut file = File::create(path)?;
+    trace!(target: "csv-export", extension = ?path.extension(), file = ?file);
+
+    // Build the CSV writer and push selected records.
+    //for result in csv_reader.records() {
+    let mut count = 0;
+    file.write_all(b"Allianz DirectCall Protokoll!")?;
+    count += 1;
+
+    let dt_end: DateTime<Local> = Local::now();
+    let duration = dt_end.signed_duration_since(dt_start);
+    println!("Duration: {:#?}", duration);
+
+    trace!(target: "csv-test", record_count = ?count, duration = ?duration);
+    state = t!("state.finished", lang);
+    res = t!("csv.import.finished", lang);
+    trace!(target: "csv-test", process = ?res, state = ?state);
+
+    Ok(count)
+}
+
+/// import from csv format
+/// https://docs.rs/csv/1.1.3/csv/cookbook/index.html
+/// https://blog.burntsushi.net/csv/
+fn import(p: &mut String, data_list: &mut PolicyDataList,
+          policy_numbers: &mut HashMap<u64, PolicyCode>, lang: &String)
+          -> Result<u64, Box<dyn Error>> {
+    use std::fs::File;
+    use std::path::Path;
+    use std::ffi::OsStr;
+
+    let mut res = t!("csv.import.started", lang);
+    let mut state = t!("state.started", lang);
+    let dt_start: DateTime<Local> = Local::now();
+
+    trace!(target: "csv-test",
+           process = ?res,
+           state = ?state,
+           date_start = ?dt_start.to_string());
+
+    // Note: slash syntax also workd on Windows!
+    let path = Path::new(p);
+
+    // must be a readable file
+    trace!(target: "csv-test", path = ?path);
+    assert_eq!(path.is_file(), true);
+
+    // only accept files with '.txt' extensions
+    let extension = path.extension();
+    assert_eq!(extension, Some(OsStr::new("txt")));
+
+    // open the file
+    let file = File::open(path)?;
+
+    trace!(target: "csv-test",
+           extension = ?extension,
+           file = ?file,
+           data_list = ?data_list.name);
+
+    // Build the CSV reader and iterate over each record.
+    let mut csv_reader = csv::ReaderBuilder::new()
+        .has_headers(true)
+        .delimiter(b' ')
+        .flexible(true)
+        //.comment(Some(b'#'))
+        //.from_reader(io::stdin());
+        //.from_path(path);
+        .from_reader(file);
+    {
+        // We nest this call in its own scope because of lifetimes.
+        let headers = csv_reader.headers()?;
+        trace!(target: "csv-test", header = ?headers);
+    }
+
+    // Deserialize the input data and push result to target vector
+    let mut count = 0;
+    for result in csv_reader.deserialize() {
+        // The iterator yields Result<StringRecord, Error>, so we check the
+        // error here.
+        let record: PolicyData = result?;
+        //println!("{:?}", record);
+
+        // WIP: write to redis backend
+        // append the policy_number to the HashMap
+        policy_numbers.insert(record.policy_number, record.policy_code);
+
+        // push record as new vector elements
+        data_list.push(record);
+
+        count +=1;
+    }
+
+    let dt_end: DateTime<Local> = Local::now();
+    let duration = dt_end.signed_duration_since(dt_start);
+
+    trace!(target: "csv-test", record_count = ?count, duration = ?duration);
+
+    state = t!("state.finished", lang);
+    res = t!("csv.import.finished", lang);
+    trace!(target: "csv-test",
+           process = ?res,
+           state = ?state,
+           date_stop = ?dt_end.to_string());
+
+    Ok(count)
+}
+
+#[allow(dead_code)]
+/// validate a given policy number
+/// result will return true or false
+fn is_valid(policy_number: &u64, policy_list: &PolicyDataList,
+            policy_numbers: &mut HashMap<u64, PolicyCode>, lang: &String)
+            -> Result<bool, Box<dyn std::error::Error>> {
+
+    let mut res = t!("policy.validation.started", lang);
+    let mut state = t!("state.started", lang);
+    let dt_start: DateTime<Local> = Local::now();
+
+    trace!(target: "csv-test",
+           process = ?res,
+           state = ?state,
+           policy_number = ?policy_number,
+           policy_list = ?policy_list.name,
+           elements = ?policy_list.policy_data.len(),
+           date_start = ?dt_start.to_string());
+    //println!("Policy_number list: {:?}", policy_list.policy_data);
+
+    // WIP: testcode to collect policy_number via iterator
+    //println!("policy_number: {:?}", policy_list.policy_data[1].policy_number);
+    //println!("Policy_number list: {:?}", policy_list.policy_data);
+    // println!("policy_list: {:?} (with {:?} elements)",
+    //          policy_list.name, policy_list.policy_data.len());
+
+
+    // policy_list.into_iter()
+    //     .filter(|num| matches(w, w1))
+    //     .clone
+    //     .collect::<Vec<policy_number>>()
+    // let my_num  = policy_list.policy_data.iter()
+    //     .map(|policy_number| {
+    //         policy_number
+    //     })
+    //     .collect::<Vec<_>>();
+
+    //println!("My policy_numbers: {:?}", my_num);
+
+        // let mut my_policy_list = [
+    //     [ ("Jack", 20), ("Jane", 23), ("Jill", 18), ("John", 19), ],
+    //     [ ("Bill", 17), ("Brenda", 16), ("Brad", 18), ("Barbara", 17), ]
+    // ];
+
+    // let teams_in_score_order = teams
+    //     .iter_mut()
+    //     .map(|team| {
+    //         team.sort_by(|&a, &b| a.1.cmp(&b.1).reverse());
+    //         team
+    //     })
+    //     .collect::<Vec<_>>();
+
+    //println!("Teams: {:?}", teams_in_score_order);
+
+    // if policy_list.policy_data.iter().any(|v| v == policy_number) {
+    //     println!("{:?} contains {}", policy_list.name, policy_number);
+    // } else {
+    //     println!("{:?} doesn't contain {}", policy_list, policy_number);
+    //}
+
+    // let test: Vec<_> = vec!["one", "two", "three"];
+    // let index: usize = test.iter().enumerate().find(|&r| r.1.to_string() == "two".to_string()).unwrap().0;
+    // println!("index: {:?} -> {:?}", index, test[index]);
+
+    //let index: usize = test.iter().enumerate().find(|&r| r.policy_number == "two".to_string()).unwrap().0;
+
+    let mut result = false;
+    match policy_numbers.get(&policy_number) {
+        Some(&policy_code) => {
+            let res = t!("policy.validation.success", lang);
+            println!("policy_number: {} ({:?})",
+                     policy_number, policy_code);
+            result = true;
+            trace!(target: "csv-test",
+                   policy_number = ?policy_number,
+                   validation = ?res,
+                   policy_code = ?policy_code);
+        },
+        _ => {
+            let res = t!("policy.validation.failed", lang);
+            //println!("Noop! Number isn't valid!");
+            println!("{:?}", res);
+            trace!(target: "csv-test",
+                   policy_number = ?policy_number,
+                   validation = ?res);
+       },
+    }
+
+
+    let dt_end: DateTime<Local> = Local::now();
+    let duration = dt_end.signed_duration_since(dt_start);
+
+    res = t!("policy.validation.finished", lang);
+    state = t!("state.finished", lang);
+    trace!(target: "csv-test",
+           process = ?res,
+           state = ?state,
+           date_stop = ?dt_end.to_string(),
+           duration = ?duration);
+
+    Ok(result)
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    use dotenv::dotenv;
+    use parse_args::parse_args;
+    //use std::process;
+    //use std::sync::Arc;
+    use tracing_subscriber::fmt;
+    use viperus::Viperus;
+
+    //static DEFAULT_FILTER: &str = concat!(module_path!(), "=", "trace");
+
+    // initialize the tracing subsystem
+    // a drop in replacement for classical logging
+    // reference: https://tokio.rs/blog/2019-08-tracing/
+    let span = tracing::span!(Level::TRACE, "csv-test");
+    let _enter = span.enter();
+    let subscriber = fmt::Subscriber::builder()
+        .with_env_filter("trace")
+        //.with_max_level(tracing::Level::DEBUG)
+        .finish();
+
+    tracing::subscriber::with_default(subscriber, || {
+        // get system environment
+        let mut lang = env::var("LANG").unwrap_or("en".to_string());
+        let mut res = t!("parse.environment", lang);
+        let mut state = t!("state.started", lang);
+        trace!(target: "csv-test", message = ?res, state = ?state);
+        //debug!(message = ?res, state = ?state);
+        trace!(target: "csv-test", environment = "system", lang = ?lang);
+
+        // testing environment: read from .env file
+        dotenv().ok();
+        match envy::from_env::<Environment>() {
+            Ok(environment) => {
+                if environment.test_lang != lang { lang = environment.test_lang; }
+            },
+            Err(e) => { debug!(target: "csv-test", "{}", e); }
+        }
+        // how to handle unumplemented lang resources??
+        res = t!("parse.environment", lang);
+        trace!(target: "csv-test", environment = "envy", lang = ?lang);
+        state = t!("state.finished", lang);
+        trace!(target: "csv-test", message = ?res, state = ?state);
+
+        // initialize viperus structure
+        let mut v = Viperus::new();
+
+        // parse commandline arguments
+        res = t!("parse.arguments", lang);
+        state = t!("state.started", lang);
+        trace!(target: "csv-test", process = ?res, state = ?state);
+
+        let _ = parse_args(&mut v);
+        state = t!("state.finished", lang);
+        trace!(target: "csv-test", process = ?res, state = ?state);
+        //trace!(target: "Viperus", "Config results: {:?}", v);
+
+        // main tasks
+        res = t!("main.started", lang);
+        state = t!("state.started", lang);
+        trace!(target: "csv-test", process = ?res, state = ?state);
+
+        // create policy structures
+        let policy_list = PolicyList::new("Allianz Versicherungsnummen-Liste");
+        println!("Policy List {:?} ", policy_list.name);
+
+        let mut policy_data = PolicyDataList::new("Allianz-Import 20200628");
+        println!("Policy Data List {:?} ", policy_data.name);
+
+        let mut policy_numbers : HashMap<u64, PolicyCode> = HashMap::new();
+
+        let mut csv_import_path = v.get::<String>("import_file").unwrap();
+        match import(&mut csv_import_path, &mut policy_data,
+                     &mut policy_numbers, &lang) {
+            Ok(count) => {
+                println!("Imported {:?} records", count);
+            }
+            Err(err) => {
+                println!("error running Csv-Test: {}", err);
+                process::exit(1);
+            }
+        }
+
+        // test if policy_number is_valid
+        // type conversion (viperus String -> u64)
+        let test_policy_number = v.get::<String>("test_policy_number").unwrap().parse::<u64>().unwrap();
+        trace!(target: "csv-test", test_policy_number = ?test_policy_number);
+        //match is_valid(&policy_number, &policy_data, &mut policy_numbers, &lang) {
+        //    Ok(true) => {
+        // use Hashmap method 'get' to check if we have the given key
+        match policy_numbers.get(&test_policy_number) {
+            Some(&policy_code) => {
+                let res = t!("policy.validation.success", lang);
+                println!("{:?}", res);
+                println!("policy_number: {} ({:?})",
+                         test_policy_number, policy_code);
+            }
+            _ => {
+                let res = t!("policy.validation.failed", lang);
+                println!("{:?}", res);
+                //println!("Nuup! Number isn't valid!");
+            },
+        }
+
+        // export policy code elements to csv-file
+        let mut csv_export_path = v.get::<String>("export_file").unwrap();
+        match export(&mut csv_export_path, &lang) {
+            Ok(count) => {
+                println!("Exported {:?} records", count);
+            }
+            Err(err) => {
+                println!("error running CSV-Export: {}", err);
+                process::exit(1);
+            }
+        }
+
+        state = t!("state.finished", lang);
+        res = t!("main.finished", lang);
+        trace!(target: "csv-test", process = ?res, state = ?state);
+    });
+
+    Ok(())
+}
+
+#[test]
+fn test_policy_number() {
+    // Takes a reference and returns Option<&V>
+    let my_policy_numbers : [u64; 2] = [1511111111, 9999999993];
+    assert_eq!(my_policy_numbers, [1511111111, 9999999993]);
+
+    //let mut csv_import_path = v.get::<String>("import_file").unwrap();
+    let mut csv_import_path = String::from("data/POLLFNR_TEST.txt");
+
+    let mut policy_data = PolicyDataList::new("PolicyDataList");
+    let mut policy_numbers : HashMap<u64, PolicyCode> = HashMap::new();
+    let lang = "en".to_string();
+
+    println!("import with Path: {:?} PolicyData: {:?} PolicyNumbers: {:?}, Lang: {:?}",
+             csv_import_path, policy_data, policy_numbers, lang);
+    let count = import(&mut csv_import_path, &mut policy_data,
+                       &mut policy_numbers, &lang);
+
+    assert_eq!(count.unwrap(), 15498);
+
+}

advotracker_client/examples/csv-test/parse_args.rs

@@ -0,0 +1,216 @@
+/*
+ * advotracker - Hotline tackingtool for Advocats
+
+ * Copyright 2020 Ralf Zerres <ralf.zerres@networkx.de>
+ * SPDX-License-Identifier: (0BSD or MIT)
+ */
+
+// parse CLI commandline arguments with clap
+use clap::{crate_authors, crate_description, crate_name, crate_version, App, Arg};
+
+//use log::{debug, info, trace, warn};
+use std::env;
+use tracing::trace;
+use viperus::Viperus;
+
+/// Parse the commandline arguments and preset default values
+/// Precedence: defaults -> config-file -> environment -> commandline
+pub fn parse_args(v: &mut Viperus) -> Result<(), Box<dyn std::error::Error>> {
+    if cfg!(feature = "fmt-clap") {
+        trace!(target: "Viperus", "Viperus feature 'fmt-clap' enabled.");
+        println!("Using feature fmt-clap");
+    }
+
+    // preset default key/value pairs (lowest priority)
+    v.add_default("config_file", String::from("csv_import.ron"));
+    v.add_default("import_file", String::from("POLLFNR_WOECHENTLICH.txt"));
+    v.add_default("export_file", String::from(""));
+    v.add_default("test_policy_number", String::from("9999999992"));
+    v.add_default("to_email_address_file", String::from("Allianz RA-Hotline <smr-rahotline@allianz.de>"));
+    v.add_default("from_email_address_file", String::from("Allianz-Hotline RA-Hiedemann <azhotline@hiedemann.de>"));
+    //v.add_default("username", String::from("nctalkbot"));
+    //v.add_default("password", String::from("botpassword"));
+    v.add_default("verbose", 0);
+
+
+    // CLI arguments are defined inline
+    let matches = App::new("csv-test")
+        .name(crate_name!())
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about(crate_description!())
+        .after_help("
+Testprogramm: Allianz Online-Beratung Im/Export CSV-Daten
+Direct-Call via IVR-System (Interactive Voice Response)
+SMR Deckungssummen-Prüfung: 089 92529 60211
+SMR Unerledigt: 089 92529 60222")
+        .template(
+            "\
+            {bin} v{version}
+{about}
+
+{all-args}
+
+(C) 2020 {author}
+{after-help}",
+        )
+        .arg(
+            Arg::with_name("configFile")
+                .short("c")
+                .long("configFile")
+                .value_name("FILE")
+                .help("Select a config file")
+                .default_value("csv_import.ron")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("importFile")
+                .short("i")
+                .long("importFile")
+                .help("Select source file for the csv-import")
+                .default_value("POLLFNR_WOECHENTLICH.txt")
+                //.default_value("POLLFNR_MINI.txt")
+               .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("exportFile")
+                .short("e")
+                .long("exportFile")
+                .help("Select target file for the csv-export")
+                .default_value("RA-Hiedemann_DirectCall.txt")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("fromEmailAddress")
+                .short("f")
+                .long("fromEmailAddress")
+                .help("Select the sender email-address (From:)")
+                .default_value("Allianz-Hotline RA-Hiedemann <azhotline@hiedemann.de>")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("toEmailAddress")
+                .short("t")
+                .long("toEmailAddress")
+                .help("Select the target email-address (To:)")
+                .default_value("Allianz RA-Hotline <smr-rahotline@allianz.de>")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("testPolicyNumber")
+                .short("p")
+                .long("testPolicyNumber")
+                .help("Test validity of given policy number")
+                //.default_value("")
+                .takes_value(true),
+        )
+        // .arg(
+        //     Arg::with_name("username")
+        //         .short("u")
+        //         .long("username")
+        //         .help("Sets username")
+        //         .takes_value(true),
+        // )
+        // .arg(
+        //     Arg::with_name("password")
+        //         .short("P")
+        //         .long("password")
+        //         .help("Sets password")
+        //         .takes_value(true),
+        // )
+        .arg(
+            Arg::with_name("verbose")
+                .short("v")
+                .long("verbose")
+                .help("Sets verbosity level")
+                .multiple(true),
+        )
+        .get_matches();
+
+    if matches.occurrences_of("verbose") > 0 {
+        // clap is using i64, viperus i32
+        let n = matches.occurrences_of("verbose") as i32;
+        v.add("verbose", n);
+    }
+
+    // preset the prefix for relevant environment variables ("ADVOTRACKER_")
+    let mut env_prefix: String = crate_name!().to_uppercase();
+    env_prefix.push_str("_");
+    v.set_env_prefix(&env_prefix);
+
+    // respect dotenv environment (e.g for testing)
+    // -> overwrites the preset default values
+    println!(
+        "RUST_LOG={}",
+        dotenv::var("RUST_LOG").unwrap_or_else(|_| String::from("None"))
+    );
+
+    // enable caching and automatic update of environment values
+    v.cache(true);
+    v.automatic_env(true);
+
+    // load user selected call arguments
+    // -> overwrites values given via environment variables
+    v.load_clap(matches)?;
+
+    // bond the clap names to camel_case rust variable names
+    v.bond_clap("configFile", "config_file");
+    v.bond_clap("importFile", "import_file");
+    v.bond_clap("exportFile", "export_file");
+    v.bond_clap("toEmailAddress", "to_email_address");
+    v.bond_clap("fromEmailAddress", "from_email_address");
+    v.bond_clap("testPolicyNumber", "test_policy_number");
+    //v.bond_clap("username", "username");
+    //v.bond_clap("password", "password");
+    v.bond_clap("verbose", "verbose");
+
+    trace!("verbose {:?}", v.get::<i32>("verbose").unwrap());
+    if v.get::<i32>("verbose").unwrap() > 0 {
+        println!(
+            "config_file: {:?}",
+            v.get::<String>("config_file").unwrap_or_default()
+        );
+        println!(
+            "import_file: {:?}",
+            v.get::<String>("import_file").unwrap_or_default()
+        );
+        println!(
+            "export_file: {:?}",
+            v.get::<String>("export_file").unwrap_or_default()
+        );
+        println!(
+            "to_email_address: {:?}",
+            v.get::<String>("to_email_address").unwrap_or_default()
+        );
+        println!(
+            "from_email_address: {:?}",
+            v.get::<String>("from_email_address").unwrap_or_default()
+        );
+        println!(
+            "test_policy_number: {:?}",
+            v.get::<String>("test_policy_number").unwrap_or_default()
+        );
+        // println!(
+        //     "username: {:?}",
+        //     v.get::<String>("username").unwrap_or_default()
+        // );
+        // println!(
+        //     "password: {:?}",
+        //     v.get::<String>("password").unwrap_or_default()
+        // ); // only for testing now
+
+        println!(
+            "verbosity level: {:?}",
+            v.get::<i32>("verbose").unwrap_or_default()
+        );
+    }
+
+    if v.get::<i32>("verbose").unwrap() > 1 {
+        println!("\nEnvironment:");
+        for (key, value) in env::vars() {
+            println!("{}={}", key, value);
+        }
+    }
+
+    Ok(())
+}