// SPDX-License-Identifier: (0BSD or MIT) /* * advotracker - Hotline tackingtool for Advocats * * Copyright 2020-2021 Ralf Zerres */ use chrono::{Local, DateTime}; use std::error::Error; use std::collections::HashMap; use std::time::{Duration, SystemTime}; use tracing::trace; //use crate::db::redis; use crate::{ data::structures::{PolicyCode, PolicyDataList, PolicyData}, Lang, }; /// import AllianzDirectCall data from a csv delimeted file /// save records to redis backend /// https://docs.rs/csv/1.1.3/csv/cookbook/index.html /// https://blog.burntsushi.net/csv/ pub fn import(p: &mut String, data_list: &mut PolicyDataList, policy_numbers: &mut HashMap, policy_data_count: &mut u64) -> Result<(u64, Duration), Box> { use std::fs::File; use std::path::Path; use std::ffi::OsStr; let lang = Lang::De(""); let mut res = t!(csv_import_started => lang); let mut state = t!(state_started => lang); let time_start = SystemTime::now(); let datetime: DateTime = time_start.into(); trace!(target: "csv-import", process = ?res, state = ?state, date_start = ?datetime.to_string()); // Note: slash syntax also works on Windows! let path = Path::new(p); // must be a readable file trace!(target: "csv-import", path = ?path); assert_eq!(path.is_file(), true); // only accept files with '.txt' extensions let extension = path.extension(); assert_eq!(extension, Some(OsStr::new("txt"))); // open the file let file = File::open(path)?; trace!(target: "csv-import", extension = ?extension, file = ?file); // Build the CSV reader let mut csv_reader = csv::ReaderBuilder::new() .has_headers(true) .delimiter(b' ') //.comment(Some(b'#')) //.flexible(true) .from_reader(file); { // We nest this call in its own scope because of lifetimes. let headers = csv_reader.headers()?; trace!(target: "csv-import", header = ?headers); } // Iterate over each record, deserialize und write to our structures let mut count : u64 = 0; for result in csv_reader.deserialize() { // The iterator yields Result, so we check the // error here. let record: PolicyData = result?; //if verbose > 3 { // println!("{:?}", record); //} // WIP: write to redis backend // for now: append the policy_number to the HashMap policy_numbers.insert(record.policy_number, record.policy_code); // push record as new vector elements data_list.push(record); count +=1; *policy_data_count = count; }; let time_end = SystemTime::now(); let duration = time_end.duration_since(time_start) .expect("Clock may have gone backwards"); trace!(target: "csv-import", record_count = ?count, duration = ?duration); state = t!(state_finished => lang); res = t!(csv_import_finished => lang); let datetime: DateTime = time_end.into(); trace!(target: "csv-import", process = ?res, state = ?state, date_stop = ?datetime.to_string()); Ok((count, duration)) } #[test] fn test_import() { // Takes a reference and returns Option<&V> let my_policy_numbers : [u64; 2] = [1511111111, 9999999993]; assert_eq!(my_policy_numbers, [1511111111, 9999999993]); let mut csv_import_path = String::from("data/POLLFNR_TEST.txt"); let mut policy_data = PolicyDataList::new("PolicyDataList"); let mut policy_numbers : HashMap = HashMap::new(); let mut policy_data_count: u64 = 0; let lang = "en".to_string(); match import(&mut csv_import_path, &mut policy_data, &mut policy_numbers, &mut policy_data_count, &lang) { Ok((count, duration)) => { println!("import {:?} records. Duration: {:?}", count, duration); } Err(err) => { println!("error running CSV-Import: {}", err); } }; for p in &my_policy_numbers { match policy_numbers.get(&p) { Some(policy_code) => { println!("Test: Policy-Number {:?} => Policy-Type {:?}, as expected!", p, policy_code); }, _ => println!("Test: Policy-Number {:?} => not valid, can't dereference the Policy-Type as expected", p), } } }