From b84289e25cd5e1cacb8f355ef55a44894dff049b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 16 Jul 2018 10:32:07 +0200 Subject: [PATCH] Add ESTER students import --- common/urls.py | 1 + .../test_files/CLOEE2_Export_Ester_2018.xls | Bin 0 -> 11776 bytes stages/tests.py | 55 +++++++++- stages/views/imports.py | 102 +++++++++++++----- templates/admin/index.html | 3 +- 5 files changed, 133 insertions(+), 28 deletions(-) create mode 100644 stages/test_files/CLOEE2_Export_Ester_2018.xls diff --git a/common/urls.py b/common/urls.py index b8adb9d..950aa81 100644 --- a/common/urls.py +++ b/common/urls.py @@ -15,6 +15,7 @@ urlpatterns = [ path('admin/', admin.site.urls), path('import_students/', views.StudentImportView.as_view(), name='import-students'), + path('import_students_ester/', views.imports.StudentEsterImportView.as_view(), name='import-students-ester'), path('import_hp/', views.HPImportView.as_view(), name='import-hp'), path('import_hp_contacts/', views.HPContactsImportView.as_view(), name='import-hp-contacts'), diff --git a/stages/test_files/CLOEE2_Export_Ester_2018.xls b/stages/test_files/CLOEE2_Export_Ester_2018.xls new file mode 100644 index 0000000000000000000000000000000000000000..19b7d401dd03833a3ea3d87cd8de27b7a408d1be GIT binary patch literal 11776 zcmeHNUu;`f8UOCJojA=O+ew^c?wYxkO4Gk;*GcU(jmAxGSWuiqu{#DTO5Mb1vpRO< zxTupD%LYR{Py`IY%NU!~K1^t%2&p`5tnJ$#hE`461AFQ~0##Jtp;a(xt@-`V@vXh& zh9wspX(eBB&OP6EzH`p+eCIp=?)BBTL)TvVb>stCLJx^YZkL**p-OHb-D&edkr2|9 z-7b|%wiwBE`W#u{RvD`Y1L#HYA#lta5&Q^E2pbTZ5n2#h5!w&}2yg*4_8k9?ayndo zevVA=e+_j)a`-DqR4mEjnU^Opf~%)4cuchcS5vZjy;zO@XX&iBU$MYv*^2qq_HXiI z7B~3hwbr*d<<;E(Kuo#dy@FWej|fL)o@>KEDl--RU(sQOA!QK5Wo$AJBW1|*khaOM zl=O8KKP>Ig0QF1sph_6>kZE`7%Fn^3QFgCPwmo-hI~147x33`IwSs)d3i2jLxxhIL*{pY0 z)wIT=<#2ZHy36Fy?{Shx^)9|lAN4HLKd2c4YM zBO7T7gQt(Oto0#g{28HJt{8ppuJ{wiPNp`_R z8<)_OU!FE_$njssq9Oi%>HUxf#xci_TicqbN0Q4V3GRJ(?(|CXQGDrrH{hO^%EW#72gb7BWXtbMr7GT1;Her{+)2X3GbZQTy}b)1S8)k`+=5c&MAKw0f{p4 zj7c)luILRc_^A!71*1L|xJD<(o*|Y^(4dT-zc|U3c=m zm|T<5O1=*p{MD8U--vcD1S-5A%S)uG!iQkLs~akOJGR^7RpmP{r_Xj)@;k9ZRhJLL zZjs(fewTDfqzZpGX7P)mNmG-p7AWi?03vy|pMSSi@D=Df(Gk7&*Zju$1(HGt!?_J-VxxDlMwU#P6zj$F(f zXcJF|((Nqgg~8D)b-i8Sc>OX=*4_>7Tkv?|4a*C!Xf66fK-I)D^s9ins%g#xYjXnW za$X#GCM;9cZUBEkUFNiA12|e+Yc_&on>x=Aj@P~4VeAbcib(r;6Y)=9HqoZmX@&;J z<_+beEm*es*H+GDdG)dYJu#rSv0PUO@)xWK;zu9IZ9?rTzXSQY)=uPboz`3!9Iu(0 z>jKAlskv@&oT0?eF$_!oqV6fzfB@{EH921b$_H4^#0Xa`WMYIPa(mb=M+?#Gbe*&b zM@*Niwu$Lz6BFYQP`eFnVj@)AgeU@N&8{y9ZX-C_*o3VDvg2>;@0b5{q|wd4cFTG85g z>1EDn;c+!b>v3sYq9k!8b4Kf_16r?3v_wf_ac8tRBG%ltK9^`&jKsO%jMi5Nv<)uN z5+#Z2x-(j=%{8}eqf4|zN#aiFjJB~3X#FnH5+#YNsxw+#X=-lUCYNZ5lBCWSDz$XI zZK?y>4KC3VC5bzRvu$ziskv>NU7{sQQsu(B)V9raK-=OHEm4wqMs&6Z>!a#V}cc!x{0L`mYk&>3w<9nf~V zL`#$;-dml~;;vJ3+lF1DB}x+S)y`b--8+G9lR!T9LUqme6G{oYXJiz?7_@o4}*UF zy67^-9W=xjlJVK8d~P8(Q;1q;rqY?{g9mmeQ^(Wzz@yyc@a)XYL8a|7g8$j?s9e=> z8&jXJy_XXG`r5O<_{Ya%CxgHIkzaOh`^_KrLiQqp{ZgSHY5M#l2)t%aBXAL!N8r`y zn+Uvgd=G)g`JW)LrR%puy<+7CnMf}b()s96CiRWkRJ3m>k=U;CsCNLNA7Sxk;eD=R zOxx6t#NrQLF}L3HuEBKSU&RL>zxQ3jUK6d?mY>$<2ltcvxjw_HiXSve)S@u|O?<|E2tA$r|EhCLY@R>13d<;MZq)IVH!{@6b+Z z;TmFoE@4%i$6Ui!Q_2Lt6a;acy`Gy68XG|3bO_T0GWjf1aso(XL&`f7AZI0sl*CJOBUy literal 0 HcmV?d00001 diff --git a/stages/tests.py b/stages/tests.py index 8cb8c0a..396d903 100644 --- a/stages/tests.py +++ b/stages/tests.py @@ -438,7 +438,7 @@ class ImportTests(TestCase): for f in os.listdir(bulletins_dir): os.remove(os.path.join(bulletins_dir, f)) - def test_import_students(self): + def test_import_students_EPC(self): """ Import CLOEE file for FE students (ASAFE, ASEFE, ASSCF, EDE, EDS) version 2018!! @@ -512,6 +512,59 @@ class ImportTests(TestCase): stud_arch = Student.objects.get(ext_id=44444) self.assertTrue(stud_arch.archived) + def test_import_students_ESTER(self): + """ + Import CLOEE file for ESTER students (MP_*) version 2018 + + Student : + - S. Lampion, 1MPTS ASE1 - Généraliste + - B. Castafiore, 1MPTS ASE1 + + Export CLOEE: + - S. Lampion, 2MPTS ASE1 - Accompagnement des enfants + - T. Tournesol, 2MPS ASSC1 + + Results in Student: + - S. Lampion, 2MPTS ASE1 (Student data + Cloee klass) + - T. Tournesol, 2MPS ASSC1 (Candidate data + Cloee klass) + - B. Castafiore, archived + """ + lev1 = Level.objects.create(name='1') + lev2 = Level.objects.create(name='2') + mp_ase = Section.objects.create(name='MP_ASE') + mp_assc = Section.objects.create(name='MP_ASSC') + Option.objects.create(name='Accompagnement des enfants') + Option.objects.create(name='Généraliste') + k1 = Klass.objects.create(name='1MPTS ASE1', section=mp_ase, level=lev1) + k2 = Klass.objects.create(name='2MPTS ASE1', section=mp_ase, level=lev2) + + # Existing students, klass should be changed or student archived. + Student.objects.create(ext_id=11111, first_name="Séraphin", last_name="Lampion", city='Marin', klass=k1) + Student.objects.create(ext_id=22222, first_name="Bianca", last_name="Castafiore", city='Marin', klass=k1) + + path = os.path.join(os.path.dirname(__file__), 'test_files', 'CLOEE2_Export_Ester_2018.xls') + self.client.login(username='me', password='mepassword') + with open(path, 'rb') as fh: + response = self.client.post(reverse('import-students-ester'), {'upload': fh}, follow=True) + msg = "\n".join(str(m) for m in response.context['messages']) + self.assertIn("La classe '2MPS ASSC1' n'existe pas encore", msg) + + k3 = Klass.objects.create(name='2MPS ASSC1', section=mp_assc, level=lev2) + with open(path, 'rb') as fh: + response = self.client.post(reverse('import-students-ester'), {'upload': fh}, follow=True) + msg = "\n".join(str(m) for m in response.context['messages']) + self.assertIn("Objets créés : 1", msg) + self.assertIn("Objets modifiés : 1", msg) + + # Student already existed, klass changed + student1 = Student.objects.get(ext_id=11111) + self.assertEqual(student1.klass.name, '2MPTS ASE1') + self.assertEqual(student1.option_ase.name, 'Accompagnement des enfants') + self.assertEqual(student1.city, 'Le Locle') + # Castafiore was archived + stud_arch = Student.objects.get(ext_id=22222) + self.assertTrue(stud_arch.archived) + def test_import_hp(self): teacher = Teacher.objects.create( first_name='Jeanne', last_name='Dupond', birth_date='1974-08-08' diff --git a/stages/views/imports.py b/stages/views/imports.py index e4ecb8f..c8f12a2 100644 --- a/stages/views/imports.py +++ b/stages/views/imports.py @@ -4,6 +4,7 @@ import tempfile from collections import OrderedDict from datetime import datetime +from fnmatch import fnmatch from subprocess import PIPE, Popen, call from tabimport import CSVImportedFile, FileFactory @@ -72,7 +73,7 @@ class ImportViewBase(FormView): class StudentImportView(ImportViewBase): - title = "Importation étudiants" + title = "Importation étudiants EPC" form_class = StudentImportForm # Mapping between column names of a tabular file and Student field names student_mapping = { @@ -81,6 +82,7 @@ class StudentImportView(ImportViewBase): 'ELE_PRENOM': 'first_name', 'ELE_RUE': 'street', 'ELE_NPA_LOCALITE': 'city', # pcode is separated from city in prepare_import + 'ELE_CODE_CANTON': 'district', 'ELE_TEL_PRIVE': 'tel', 'ELE_TEL_MOBILE': 'mobile', 'ELE_EMAIL_RPN': 'email', @@ -102,12 +104,14 @@ class StudentImportView(ImportViewBase): } mapping_option_ase = { 'GEN': 'Généraliste', + 'Enfants': 'Accompagnement des enfants', 'ENF': 'Accompagnement des enfants', 'HAN': 'Accompagnement des personnes handicapées', 'PAG': 'Accompagnement des personnes âgées', } # Those values are always taken from the import file fields_to_overwrite = ['klass', 'login_rpn'] + klasses_to_skip = [] def get_form_kwargs(self): kwargs = super().get_form_kwargs() @@ -142,6 +146,27 @@ class StudentImportView(ImportViewBase): values['option_ase'] = None return values + @property + def _existing_students(self): + return Student.objects.filter( + archived=False, + ext_id__isnull=False, + klass__section__in=[s for s in Section.objects.all() if s.is_EPC] + ) + + def update_defaults_from_candidate(self, defaults): + # Any DoesNotExist exception will bubble up. + candidate = Candidate.objects.get(last_name=defaults['last_name'], + first_name=defaults['first_name']) + # Mix CLOEE data and Candidate data + if candidate.option in self.mapping_option_ase: + defaults['option_ase'] = Option.objects.get(name=self.mapping_option_ase[candidate.option]) + if candidate.corporation: + defaults['corporation'] = candidate.corporation + defaults['instructor'] = candidate.instructor + defaults['dispense_ecg'] = candidate.exemption_ecg + defaults['soutien_dys'] = candidate.handicap + def import_data(self, up_file): """ Import Student data from uploaded file. """ @@ -151,12 +176,8 @@ class StudentImportView(ImportViewBase): obj_created = obj_modified = 0 err_msg = [] seen_students_ids = set() - fe_students_ids = set( - Student.objects.filter( - archived=False, - ext_id__isnull=False, - klass__section__in=[s for s in Section.objects.all() if s.is_EPC] - ).values_list('ext_id', flat=True) + existing_students_ids = set( + self._existing_students.values_list('ext_id', flat=True) ) for line in up_file: @@ -166,12 +187,20 @@ class StudentImportView(ImportViewBase): if student_defaults['ext_id'] in seen_students_ids: # Second line for student, ignore it continue + for klass in self.klasses_to_skip: + if fnmatch(student_defaults['klass'], klass): + continue seen_students_ids.add(student_defaults['ext_id']) - corporation_defaults = { - val: strip(line[key]) for key, val in self.corporation_mapping.items() - } - student_defaults['corporation'] = self.get_corporation(corporation_defaults) + if self.corporation_mapping: + corporation_defaults = { + val: strip(line[key]) for key, val in self.corporation_mapping.items() + } + student_defaults['corporation'] = self.get_corporation(corporation_defaults) + + if 'option_ase' in self.fields_to_overwrite: + if student_defaults['option_ase'] in self.mapping_option_ase: + student_defaults['option_ase'] = self.mapping_option_ase[student_defaults['option_ase']] defaults = self.clean_values(student_defaults) try: @@ -189,18 +218,7 @@ class StudentImportView(ImportViewBase): obj_modified += 1 except Student.DoesNotExist: try: - candidate = Candidate.objects.get(last_name=defaults['last_name'], - first_name=defaults['first_name']) - # Mix CLOEE data and Candidate data - if candidate.option in self.mapping_option_ase: - defaults['option_ase'] = Option.objects.get(name=self.mapping_option_ase[candidate.option]) - if candidate.corporation: - defaults['corporation'] = candidate.corporation - defaults['instructor'] = candidate.instructor - defaults['dispense_ecg'] = candidate.exemption_ecg - defaults['soutien_dys'] = candidate.handicap - Student.objects.create(**defaults) - obj_created += 1 + self.update_defaults_from_candidate(defaults) except Candidate.DoesNotExist: # New student with no matching Candidate err_msg.append('Étudiant non trouvé dans les candidats: {0} {1} - classe: {2}'.format( @@ -208,12 +226,12 @@ class StudentImportView(ImportViewBase): defaults['first_name'], defaults['klass']) ) - Student.objects.create(**defaults) - obj_created += 1 + Student.objects.create(**defaults) + obj_created += 1 # Archive students who have not been exported - rest = fe_students_ids - seen_students_ids + rest = existing_students_ids - seen_students_ids archived = 0 for student_id in rest: st = Student.objects.get(ext_id=student_id) @@ -237,6 +255,38 @@ class StudentImportView(ImportViewBase): return corp +class StudentEsterImportView(StudentImportView): + title = "Importation étudiants ESTER" + # Mapping between column names of a tabular file and Student field names + student_mapping = { + 'ELE_NUMERO': 'ext_id', + 'ELE_NOM': 'last_name', + 'ELE_PRENOM': 'first_name', + 'ELE_RUE': 'street', + 'ELE_NPA_LOCALITE': 'city', # pcode is separated from city in prepare_import + 'ELE_DATE_NAISSANCE': 'birth_date', + 'ELE_AVS': 'avs', + 'ELE_SEXE': 'gender', + 'INS_CLASSE': 'klass', + 'PROF_DOMAINE_SPEC': 'option_ase', + } + corporation_mapping = None + # Those values are always taken from the import file + fields_to_overwrite = ['klass', 'street', 'city', 'option_ase'] + klasses_to_skip = ['1CMS*'] # Abandon classes 1CMS ASE + 1CMS ASSC + + @property + def _existing_students(self): + return Student.objects.filter( + archived=False, + ext_id__isnull=False, + klass__section__in=[s for s in Section.objects.all() if s.is_ESTER] + ) + + def update_defaults_from_candidate(self, defaults): + pass + + class HPImportView(ImportViewBase): """ Importation du fichier HyperPlanning pour l'établissement des feuilles diff --git a/templates/admin/index.html b/templates/admin/index.html index 23a7077..92d8cdc 100644 --- a/templates/admin/index.html +++ b/templates/admin/index.html @@ -84,7 +84,8 @@ document.addEventListener("DOMContentLoaded", function(event) {