135 lines
4.4 KiB
Rust
135 lines
4.4 KiB
Rust
// SPDX-License-Identifier: (0BSD or MIT)
|
|
/*
|
|
* advotracker - Hotline tackingtool for Advocats
|
|
*
|
|
* Copyright 2020-2021 Ralf Zerres <ralf.zerres@networkx.de>
|
|
*/
|
|
|
|
use chrono::{Local, DateTime};
|
|
use std::error::Error;
|
|
use std::collections::HashMap;
|
|
use std::time::{Duration, SystemTime};
|
|
use tracing::trace;
|
|
|
|
//use crate::db::redis;
|
|
use crate::{
|
|
data::structures::{PolicyCode, PolicyDataList, PolicyData},
|
|
Lang,
|
|
};
|
|
|
|
/// import AllianzDirectCall data from a csv delimeted file
|
|
/// save records to redis backend
|
|
/// https://docs.rs/csv/1.1.3/csv/cookbook/index.html
|
|
/// https://blog.burntsushi.net/csv/
|
|
pub fn import(p: &mut String, data_list: &mut PolicyDataList,
|
|
policy_numbers: &mut HashMap<u64, PolicyCode>,
|
|
policy_data_count: &mut u64)
|
|
-> Result<(u64, Duration), Box<dyn Error>> {
|
|
use std::fs::File;
|
|
use std::path::Path;
|
|
use std::ffi::OsStr;
|
|
|
|
let lang = Lang::De("");
|
|
let mut res = t!(csv_import_started => lang);
|
|
let mut state = t!(state_started => lang);
|
|
let time_start = SystemTime::now();
|
|
let datetime: DateTime<Local> = time_start.into();
|
|
|
|
trace!(target: "csv-import", process = ?res, state = ?state, date_start = ?datetime.to_string());
|
|
|
|
// Note: slash syntax also works on Windows!
|
|
let path = Path::new(p);
|
|
|
|
// must be a readable file
|
|
trace!(target: "csv-import", path = ?path);
|
|
assert_eq!(path.is_file(), true);
|
|
|
|
// only accept files with '.txt' extensions
|
|
let extension = path.extension();
|
|
assert_eq!(extension, Some(OsStr::new("txt")));
|
|
|
|
// open the file
|
|
let file = File::open(path)?;
|
|
trace!(target: "csv-import", extension = ?extension, file = ?file);
|
|
|
|
// Build the CSV reader
|
|
let mut csv_reader = csv::ReaderBuilder::new()
|
|
.has_headers(true)
|
|
.delimiter(b' ')
|
|
//.comment(Some(b'#'))
|
|
//.flexible(true)
|
|
.from_reader(file);
|
|
{
|
|
// We nest this call in its own scope because of lifetimes.
|
|
let headers = csv_reader.headers()?;
|
|
trace!(target: "csv-import", header = ?headers);
|
|
}
|
|
|
|
// Iterate over each record, deserialize und write to our structures
|
|
let mut count : u64 = 0;
|
|
for result in csv_reader.deserialize() {
|
|
// The iterator yields Result<StringRecord, Error>, so we check the
|
|
// error here.
|
|
let record: PolicyData = result?;
|
|
//if verbose > 3 {
|
|
// println!("{:?}", record);
|
|
//}
|
|
|
|
// WIP: write to redis backend
|
|
// for now: append the policy_number to the HashMap
|
|
policy_numbers.insert(record.policy_number, record.policy_code);
|
|
|
|
// push record as new vector elements
|
|
data_list.push(record);
|
|
|
|
count +=1;
|
|
*policy_data_count = count;
|
|
};
|
|
|
|
let time_end = SystemTime::now();
|
|
let duration = time_end.duration_since(time_start)
|
|
.expect("Clock may have gone backwards");
|
|
trace!(target: "csv-import", record_count = ?count, duration = ?duration);
|
|
|
|
state = t!(state_finished => lang);
|
|
res = t!(csv_import_finished => lang);
|
|
let datetime: DateTime<Local> = time_end.into();
|
|
trace!(target: "csv-import", process = ?res, state = ?state, date_stop = ?datetime.to_string());
|
|
|
|
Ok((count, duration))
|
|
}
|
|
|
|
#[test]
|
|
fn test_import() {
|
|
// Takes a reference and returns Option<&V>
|
|
let my_policy_numbers : [u64; 2] = [1511111111, 9999999993];
|
|
assert_eq!(my_policy_numbers, [1511111111, 9999999993]);
|
|
|
|
let mut csv_import_path = String::from("data/POLLFNR_TEST.txt");
|
|
let mut policy_data = PolicyDataList::new("PolicyDataList");
|
|
let mut policy_numbers : HashMap<u64, PolicyCode> = HashMap::new();
|
|
let mut policy_data_count: u64 = 0;
|
|
let lang = "en".to_string();
|
|
|
|
match import(&mut csv_import_path, &mut policy_data,
|
|
&mut policy_numbers, &mut policy_data_count,
|
|
&lang) {
|
|
Ok((count, duration)) => {
|
|
println!("import {:?} records. Duration: {:?}", count, duration);
|
|
}
|
|
Err(err) => {
|
|
println!("error running CSV-Import: {}", err);
|
|
}
|
|
};
|
|
|
|
for p in &my_policy_numbers {
|
|
match policy_numbers.get(&p) {
|
|
Some(policy_code) => {
|
|
println!("Test: Policy-Number {:?} => Policy-Type {:?}, as expected!",
|
|
p, policy_code);
|
|
},
|
|
_ => println!("Test: Policy-Number {:?} => not valid, can't dereference the Policy-Type as expected", p),
|
|
}
|
|
}
|
|
}
|