Verified Commit bc43dc1d authored by Kaspar Vollenweider's avatar Kaspar Vollenweider
Browse files

chore(access-importer): remove completely because obsolete

And it would certainly not work any more anyway
parent 7535f59f
......@@ -35,7 +35,6 @@ gem 'jquery-rails'
gem 'jquery-ui-rails'
gem 'js-routes'
gem 'lodash-rails'
gem 'mdb'
gem 'panter-rails-deploy'
gem 'paperclip'
gem 'paranoia'
......
......@@ -276,7 +276,6 @@ GEM
mini_mime (>= 0.1.1)
marcel (0.3.3)
mimemagic (~> 0.3.2)
mdb (0.4.1)
memoist (0.16.2)
method_source (1.0.0)
mime-types (3.3.1)
......@@ -582,7 +581,6 @@ DEPENDENCIES
letter_opener_web
listen
lodash-rails
mdb
minitest
overcommit
panter-rails-deploy
......
......@@ -9,18 +9,17 @@ Ruby version: 2.4.2
## Table of content
- [AOZ Voluntary Platform](#aoz-voluntary-platform)
- [Pipeline Status](#pipeline-status)
- [Table of content](#table-of-content)
- [Dependencies](#dependencies)
- [Developer Dependencies](#developer-dependencies)
- [User seeds for development](#user-seeds-for-development)
- [Create initial superadmin account](#create-initial-superadmin-account)
- [Sort locale yaml files](#sort-locale-yaml-files)
- [Importing from access db with rake task](#importing-from-access-db-with-rake-task)
- [Run model, integration and controller tests](#run-model-integration-and-controller-tests)
- [Run system (acceptance) tests](#run-system-acceptance-tests)
- [LICENSE](#license)
* [AOZ Voluntary Platform](#aoz-voluntary-platform)
* [Pipeline Status](#pipeline-status)
* [Table of content](#table-of-content)
* [Dependencies](#dependencies)
* [Developer Dependencies](#developer-dependencies)
* [User seeds for development](#user-seeds-for-development)
* [Create initial superadmin account](#create-initial-superadmin-account)
* [Sort locale yaml files](#sort-locale-yaml-files)
* [Run model, integration and controller tests](#run-model-integration-and-controller-tests)
* [Run system (acceptance) tests](#run-system-acceptance-tests)
* [LICENSE](#license)
### Dependencies
......@@ -73,14 +72,6 @@ Run this task, in order to sort the locale files alphabetically.
$ rails i18n:sort
```
### Importing from access db with rake task
Run in the command line:
```bash
$ rails access:import file=path/to/access_file.accdb
```
### Run model, integration and controller tests
```bash
......
module AccUtils
def make_mappable(table_name, primary_key, sanitize = false)
@acdb[table_name].map do |r|
[
r[primary_key].to_i,
sanitize ? sanitize_record(r) : r
]
end.to_h
end
# normalize hash keys from access db
# they can be with uppercase and special chars
#
def down_hkeys(row)
row.transform_keys { |key| key.to_s.underscore.to_sym }
end
def parse_int_fields(record, *keys)
record.merge(record.slice(*keys).compact.transform_values(&:to_i))
end
def parse_datetime_fields(record, *keys)
record.merge(record.slice(*keys).compact.transform_values(&:to_time))
end
def parse_float_fields(record, *keys)
record.merge(record.slice(*keys).compact.transform_values(&:to_f))
end
def parse_date_fields(record, *keys)
record.merge(record.slice(*keys).compact.transform_values(&:to_date))
end
def parse_boolean_fields(record, *keys)
record.merge(
parse_int_fields(record, *keys).slice(*keys).compact.transform_values do |val|
val == 1
end
)
end
def salutation(anrede, gender = nil)
return 'mrs' if anrede == 'Frau'
return 'mr' if anrede == 'Herr'
return 'mrs' if gender == 'female'
return 'mr' if gender == 'male'
nil
end
def email(h_email)
return generate_bogus_email if h_email.nil?
email = h_email.sub(/^\#mailto:/, '').sub(/\#$/, '')
return generate_bogus_email unless email.match?(Devise.email_regexp)
email
end
def generate_bogus_email
"unknown_email_#{Time.zone.now.to_f}@example.com"
end
def map_gender(value)
return nil unless value
return 'female' if value == 'W'
return 'male' if value == 'M'
end
def contact_attributes(haupt_person)
if haupt_person[:t_Telefon1].blank? && haupt_person[:t_Telefon2].present?
haupt_person[:t_Telefon1] = haupt_person[:t_Telefon2]
haupt_person[:t_Telefon2] = nil
end
{ contact_attributes: {
first_name: haupt_person[:t_Vorname] || 'unbekannt',
last_name: haupt_person[:t_Nachname] || 'unbekannt',
street: haupt_person[:t_Adresszeile1] || 'unbekannt',
extended: haupt_person[:t_Adresszeile2],
city: haupt_person[:city] || 'unbekannt',
postal_code: haupt_person[:postal_code] || '0000',
primary_email: haupt_person[:email],
primary_phone: haupt_person[:t_Telefon1] || 'Keine Nummer für Import vorhanden',
secondary_phone: haupt_person[:t_Telefon2]
} }
end
def import_attributes(access_name, access_id, related_records)
{ import_attributes: access_import(access_name, access_id, related_records) }
end
def access_import(access_name, access_id, related_records)
{
access_id: access_id,
base_origin_entity: access_name.to_s,
store: related_records
}
end
def language_skills_attributes(sprachen)
{ language_skills_attributes: map_sprachen_to_language_skills(sprachen) }
end
def map_sprachen_to_language_skills(sprachen)
return {} if sprachen.blank?
sprachen.map do |sprache|
[(Time.now.to_f * 1000).to_i,
{
language: sprache[:language][:lang],
level: sprache[:kenntnisstufe_ve].presence || 'basic'
}]
end.to_h
end
def now
Time.zone.now
end
def birth_year(geburtsdatum, jahrgang)
return geburtsdatum if geburtsdatum
Date.parse(jahrgang + '-06-01') if jahrgang
end
end
require 'securerandom'
require 'ostruct'
class AccessImport
include AccessImportSetup
include AccessImportTransformers
def make_departments
start_message(:department)
department_transform.import_all
display_stats(Department)
end
def make_clients
start_message(:client)
client_transform.import_all
display_stats(Client)
end
def make_volunteers
start_message(:volunteer)
volunteer_transform.import_all
display_stats(Volunteer)
end
def make_assignments
start_message(:assignment)
assignment_transform.import_all
display_stats(Assignment, Volunteer, Client)
end
def make_group_offers
start_message(:group_offer)
shell_message '... from Kurse'
kurs_transform.import_all
display_stats(GroupOffer, GroupAssignment)
shell_message '... from Animation f'
group_offer_transform.import_all
display_stats(GroupOffer, GroupAssignment)
shell_message '... from Kurzeinsatz'
group_offer_transform.import_all(@freiwilligen_einsaetze.where_kurzeinsatz)
display_stats(GroupOffer, GroupAssignment)
shell_message '... from Andere'
group_offer_transform.import_all(@freiwilligen_einsaetze.where_andere)
display_stats(GroupOffer, GroupAssignment)
end
def make_journal
start_message(:journal)
Import.client.or(Import.volunteer).find_each do |import|
journal_transform.import_all(
@journale.where_haupt_person(import.store['haupt_person']['pk_Hauptperson'])
)
end
display_stats(Journal)
end
def make_hours
start_message(:hour)
Import.volunteer.find_each do |import|
hour_transform.import_all(
@stundenerfassung.where_personen_rolle(import.access_id)
)
end
display_stats(Hour)
end
# Clean up after imports finished
def self.finalize
proc { User.find_by(email: EMAIL).delete } # Remove the import user with softdelete
end
end
module AccessImportSetup
IMPORT_USER_EMAIL = 'aoz_access_importer@example.com'.freeze
IMPORT_USER_NAME = 'AOZ Import'.freeze
attr_reader :acdb
attr_reader :import_user
def initialize(path)
ObjectSpace.define_finalizer(self, self.class.finalize)
@import_user = create_or_fetch_import_user
@acdb = Mdb.open(path)
setup_class_variables(*instantiate_all_accessors)
@sprache_pro_hauptperson.add_other_accessors(@sprachen, @sprach_kenntnisse)
@einsatz_orte.add_other_accessors(@plz)
@haupt_person.add_other_accessors(@plz, @laender, @sprache_pro_hauptperson, @sprachen,
@sprach_kenntnisse)
@kontoangaben.add_other_accessors(@plz)
# don't overwrite imported accepted_at values
Volunteer.skip_callback(:save, :record_acceptance_change, if: :accepted_at?)
Client.skip_callback(:save, :record_acceptance_change, if: :accepted_at?)
end
def instantiate_all_accessors
Dir['lib/access_import/accessors/*.rb']
.map { |file| File.basename(file, '.*') }
.reject { |name| name == 'accessor' }
.map do |name|
name.camelize.constantize.new(@acdb)
end
end
def setup_class_variables(*accessors)
accessors.each do |accessor|
class_eval { attr_reader accessor.class.name.underscore.to_sym }
instance_variable_set("@#{accessor.class.name.underscore}", accessor)
end
end
# Create Import user, needed for creating records that depend on a creator user
#
def create_or_fetch_import_user
if User.with_deleted.exists?(email: IMPORT_USER_EMAIL)
return User.with_deleted.find_by(email: IMPORT_USER_EMAIL).restore
end
user = User.create!(email: IMPORT_USER_EMAIL, password: SecureRandom.hex(60), role: 'superadmin')
user.build_profile
user.profile.build_contact(first_name: IMPORT_USER_NAME, last_name: IMPORT_USER_NAME,
primary_email: IMPORT_USER_EMAIL, primary_phone: '0000', city: 'Zuerich', postal_code: '8000',
street: 'xxxxxx')
user.save!
user
end
# Shell Output methods
#
def shell_message(message)
puts message
end
def start_message(import_model)
shell_message "Start Importing #{import_model.to_s.classify.pluralize}"
end
# display amount of imports for models
def display_stats(*models)
models.each do |model|
shell_message stat_text(model)
end
end
def stat_text(model)
"Imported #{Import.where(importable_type: model.name)&.count} #{model.name.pluralize}."
end
def overall_stats
shell_message "Overall imported #{Import.count} records"
shell_message imported_stat_texts.join("\n")
end
def imported_stat_texts
[Assignment, Client, Department, GroupAssignment, GroupOfferCategory, GroupOffer, Hour, Journal,
Volunteer].map do |model|
stat_text(model)
end
end
end
# sort of singleton instance methods for the transformers
#
module AccessImportTransformers
def assignment_transform
@assignment_transform ||= AssignmentTransform.new(self, @begleitete, @freiwilligen_einsaetze,
@personen_rolle)
end
def client_transform
@client_transform ||= ClientTransform.new(self, @begleitete, @haupt_person, @familien_rollen,
@personen_rolle, @verfahrens_history)
end
def department_transform
@department_transform ||= DepartmentTransform.new(self, @einsatz_orte)
end
def einsatz_transform
@einsatz_transform ||= EinsatzTransform.new(self, @freiwilligen_einsaetze, @personen_rolle)
end
def group_offer_transform
@group_offer_transform ||= GroupOfferTransform.new(self, @freiwilligen_einsaetze, @rollen)
end
def group_assignment_transform
@group_assignment_transform ||= GroupAssignmentTransform.new(self, @begleitete,
@freiwilligen_einsaetze, @personen_rolle, @haupt_person)
end
def journal_transform
@journal_transform ||= JournalTransform.new(self, @freiwilligen_einsaetze, @journale,
@personen_rolle, @haupt_person)
end
def kurs_transform
@kurs_transform ||= KursTransform.new(self, @kurse, @begleitete, @haupt_person,
@familien_rollen, @personen_rolle, @kursarten, @freiwilligen_einsaetze, @einsatz_orte)
end
def kursart_transform
@kursart_transform ||= KursartTransform.new(self, @kursarten)
end
def volunteer_transform
@volunteer_transform ||= VolunteerTransform.new(self, @haupt_person, @personen_rolle,
@kontoangaben, @stundenerfassung)
end
def billing_expense_transform
@billing_expense_transform ||= BillingExpenseTransform.new(self, @haupt_person, @personen_rolle,
@stundenerfassung, @freiwilligen_entschaedigung)
end
def hour_transform
@hour_transform ||= HourTransform.new(self, @haupt_person, @personen_rolle, @stundenerfassung,
@freiwilligen_einsaetze)
end
end
require 'access_import/acc_utils'
require 'ostruct'
# Accessor normalizes and prepares Access tables and records to hashes
#
class Accessor
include AccUtils
# Adds the instantiating Access Import instnace plus needed other Accessors as
# class variables
#
def initialize(acdb, *other_accessors)
add_other_accessors(*other_accessors) if other_accessors.any?
@acdb = acdb
end
def add_other_accessors(*accessors)
accessors.each do |accessor|
instance_variable_set("@#{accessor.class.name.underscore}", accessor)
end
end
def all
@all ||= hash_all
end
def find(id)
all[id.to_i]
end
COST_UNIT_MAP = {
1 => 'city', 2 => 'canton', 3 => 'municipality', 4 => 'municipality', 5 => 'municipality'
}.freeze
def cost_unit(fk_kostentraeger)
COST_UNIT_MAP[fk_kostentraeger] if fk_kostentraeger&.positive?
end
ACCESS_ROLES = OpenStruct.new(volunteer: 1, client: 2, animator: 3, participant: 4).freeze
FREIWILLIGEN_FUNKTION_BY_NAME = OpenStruct.new(
begleitung: OpenStruct.new(id: 1, bezeichnung: 'Begleitung', rolle: 'Freiwillige/r'),
kurs: OpenStruct.new(id: 2, bezeichnung: 'Kurs', rolle: 'Freiwillige/r'),
animation_f: OpenStruct.new(id: 3, bezeichnung: 'Animation F', rolle: 'Freiwillige/r'),
kurzeinsatz: OpenStruct.new(id: 4, bezeichnung: 'Kurzeinsatz', rolle: 'Freiwillige/r'),
andere: OpenStruct.new(id: 5, bezeichnung: 'Andere', rolle: 'Freiwillige/r'),
animation_a: OpenStruct.new(id: 6, bezeichnung: 'Animation A', rolle: 'Animator/in')
).freeze
def freiwilligen_funktion(id)
FREIWILLIGEN_FUNKTION_BY_NAME.to_h.find { |funktion| funktion[1].id == id }.last
end
SEMESTER = {
1 => { semester: 'Frühling / Sommer', rolle: 'Animator/in' },
2 => { semester: 'Herbst / Winter', rolle: 'Animator/in' },
3 => { semester: '1. Halbjahr', rolle: 'Freiwillige/r' },
4 => { semester: '2. Halbjahr', rolle: 'Freiwillige/r' }
}.freeze
LEHRMITTEL = {
1 => 'ABC 1 - Alphabetisierung für Erwachsene',
2 => 'ABC 2 - Alphabetisierung für Erwachsene',
3 => 'Vorstufe Deutsch 1',
4 => 'Vorstufe Deutsch 2'
}.freeze
AUSBILDUNGS_TYPEN = {
1 => '<keine>',
2 => 'Primarschule',
3 => 'Sekundarschule',
4 => 'Fachmittelschule',
5 => 'Fahhochschule',
6 => 'Gymnasium',
7 => 'GEP-Einsatz',
8 => 'EBA Eidg. Berufsattest',
9 => 'eidg. Anerkannte Berufslehre'
}.freeze
JOURNAL_KATEGORIEN = {
1 => 'Telefonat',
2 => 'Gespräch',
3 => 'E-Mail',
4 => 'Rückmeldung',
5 => 'Datei'
}.freeze
end
class Ausbildungen < Accessor
def hash_all
make_mappable(:tbl_Ausbildungen, :pk_Ausbildung, true)
end
def sanitize_record(rec)
rec = parse_int_fields(rec,
:pk_Ausbildung, :fk_Hauptperson, :fk_AusbildungsTyp, :fk_FreiwilligenEinsatz)
rec[:ausbildung] = AUSBILDUNGS_TYPEN[rec[:fk_AusbildungsTyp]]
parse_datetime_fields(rec, :d_MutDatum).except(:fk_AusbildungsTyp)
end
def where_haupt_person(hauptperson_id)
all.select do |_key, ausbildung|
ausbildung[:fk_Hauptperson] == hauptperson_id.to_i
end
end
end
class Begleitete < Accessor
def hash_all
make_mappable(:tbl_Begleitete, :pk_Begleitete, true)
end
def sanitize_record(rec)
rec = parse_int_fields(rec, :fk_FamilienRolle, :fk_PersonenRolle, :pk_Begleitete, :z_Jahrgang)
rec = parse_datetime_fields(rec, :d_MutDatum)
rec[:gender] = map_gender(rec[:t_Geschlecht])
rec[:birth_year] = Date.ordinal(rec[:z_Jahrgang]) if rec[:z_Jahrgang]
rec[:relation] = map_familien_rolle(rec)
rec
end
def where_personen_rolle(personen_rolle_id)
all.select do |_key, personen_rolle|
personen_rolle[:fk_PersonenRolle] == personen_rolle_id.to_i
end
end
def find_with_personenrolle(begleitet_id)
begleitet = find(begleitet_id)
begleitet.merge(personen_rolle: @personen_rolle.find(begleitet[:fk_PersonenRolle]))
end
# :wife, :husband, :mother, :father, :daughter, :son, :sister, :brother, :aunt, :uncle
# 1: <keine>, 2: 'Hauptperson', 3: 'Ehepartner/in', 4: 'Kind', 5: 'Geschwister', 6: 'Eltern'
#
def map_familien_rolle(record)
return nil if [nil, 1, 2].include? record[:fk_FamilienRolle]
return handle_female(record) if record[:gender] == 'female'
handle_male(record) if record[:gender] == 'male'
end
def handle_female(record)
return 'mother' if record[:fk_FamilienRolle] == 6
return 'sister' if record[:fk_FamilienRolle] == 5
return 'wife' if record[:fk_FamilienRolle] == 3
'daughter' if record[:fk_FamilienRolle] == 4
end
def handle_male(record)
return 'father' if record[:fk_FamilienRolle] == 6
return 'brother' if record[:fk_FamilienRolle] == 5
return 'husband' if record[:fk_FamilienRolle] == 3
'son' if record[:fk_FamilienRolle] == 4
end
end
class EinsatzOrte < Accessor
def hash_all
make_mappable(:tbl_EinsatzOrte, :pk_EinsatzOrt, true)
end
def sanitize_record(rec)