From 26630687db8b9162a5cc0fbb2d51b835ada5576d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 19 Jul 2017 11:33:54 +0200 Subject: [PATCH] Fixed student import --- common/settings.py | 4 +- stages/models.py | 34 +------------ stages/test_files/EXPORT_GAN.xlsx | Bin 5880 -> 5931 bytes stages/tests.py | 10 ++-- stages/utils.py | 8 +++ stages/views.py | 79 +++++++++++++++++++++++------- 6 files changed, 79 insertions(+), 56 deletions(-) diff --git a/common/settings.py b/common/settings.py index c854cf9..ec5c2a0 100644 --- a/common/settings.py +++ b/common/settings.py @@ -118,7 +118,7 @@ ALLOWED_HOSTS = ['localhost', 'stages.pierre-coullery.ch'] # Mapping between column names of a tabular file and Student field names STUDENT_IMPORT_MAPPING = { - 'NO_CLOEE': 'ext_id', + 'NOCLOEE': 'ext_id', 'NOM': 'last_name', 'PRENOM': 'first_name', 'RUE': 'street', @@ -129,8 +129,6 @@ STUDENT_IMPORT_MAPPING = { 'DATENAI': 'birth_date', 'NAVS13': 'avs', 'SEXE': 'gender', - 'NO_EMPLOYEUR' : 'corporation', - 'NO_FORMATEUR' : 'instructor', 'CLASSE_ACTUELLE': 'klass', } diff --git a/stages/models.py b/stages/models.py index 9744aca..932ab42 100644 --- a/stages/models.py +++ b/stages/models.py @@ -6,14 +6,6 @@ from django.db import models from . import utils -def is_int(s): - try: - int(s) - return True - except ValueError: - return False - - class Section(models.Model): """ Filières """ name = models.CharField(max_length=20, verbose_name='Nom') @@ -156,7 +148,7 @@ class Student(models.Model): return '%d ans%s' % (age_y, ' %d m.' % age_m if age_m > 0 else '') @classmethod - def prepare_import(cls, student_values, corp_values, inst_values): + def prepare_import(cls, student_values): ''' Hook for tabimport, before new object get created ''' if 'klass' in student_values: try: @@ -165,30 +157,8 @@ class Student(models.Model): raise Exception("La classe '%s' n'existe pas encore" % student_values['klass']) student_values['klass'] = k - if 'corporation' in student_values: - if 'city' in corp_values and is_int(corp_values['city'][:4]): - corp_values['pcode'], _, corp_values['city'] = corp_values['city'].partition(' ') - if student_values['corporation'] != '': - obj, created = Corporation.objects.get_or_create( - ext_id=student_values['corporation'], - defaults = corp_values - ) - inst_values['corporation'] = obj - student_values['corporation'] = obj - else: - student_values['corporation'] = None - - if 'instructor' in student_values: - if student_values['instructor'] != '': - obj, created = CorpContact.objects.get_or_create( - ext_id=student_values['instructor'], - defaults = inst_values - ) - student_values['instructor'] = obj - else: - student_values['instructor'] = None # See if postal code included in city, and split them - if 'city' in student_values and is_int(student_values['city'][:4]): + if 'city' in student_values and utils.is_int(student_values['city'][:4]): student_values['pcode'], _, student_values['city'] = student_values['city'].partition(' ') student_values['archived'] = False return student_values diff --git a/stages/test_files/EXPORT_GAN.xlsx b/stages/test_files/EXPORT_GAN.xlsx index 9dbc23e80a898fe4bc372a37964571a1bcf61be6..9e2217380c2915ce283f69b4392b6b4289de0e18 100644 GIT binary patch delta 3988 zcmZ8kXH*l)5>5hv(0dUvbdVmZlu$*8)KEhWy>}3h8c^v1LXajXC`fMx=_n#iqy`iP z0|+8Us`MiA(EH9k_ul=pGka!cXLrAyGvA?jwFC`PkA##N0D(XN^4dF%G^{|9^BF4$ zgkdU+^-T*rA%O*?iMLwKc>-JXkIH+C+Wn2l@Xaz;)YMkpPa5}p3c2G9jn00I`Vrf zd_G+_A2u1g3eGRGcaaTX07yM@NHcAg-WmV^hz0`A^?%%h?)f6t48)DGGwOjcM<1ZX zqUYN|l(NqJPDylGc}Ww87PC}lEp?vCs=vAj{bZDMF9^Y7M50QU!P65sRkK5l*c55z zWo_VPG^Fx9({6iVA|JEmK(>QbiNh^ghY8uP>#a{aVOu_wH2v2b;;xwF1gU6TT#@bj zuub!2SQ}jeZxhiUElk9?rO7SL`_Qzq4TLF>50C~^qY_FsI434dwnj~I1*>UD2kvRm z4q55*-ZHW#7i>Y%O%3Nu=WY~#u4)oc-kTdTY@J9ie=T*m{K7x$kJoCRqD{Y3n)Q zZQ9i6W&;mK4nCS^o%`$6<6LSrRAS?5YhE}g)kv;8$aRXVy^QS;v?7mMFMV6=*|OZ%qW{;UD2ytC z{m&m{5d;AI8=bgQ61G3^#Ib>8L4R;+NX~`HGcuZmvG;GDFz0OYd0wZ|zs`=yP9|aE zxW-Ws3QNXXDulXpjBgX&2d)<|k9*`d@s_a-7@^O89)_%0?`N=q!E~009xH%Oqy%YR zADl;wYGFj|I0_PHmL!lAlt9wj%Lq+;6jhnSY`?-TT1=r&218Z-TvU)A4UGE&mBdI* z`kP2lmfkRTNf!faC(EW$e!puM_MJHi#(wOB){4*K{u=xg)H<)*&B~U%FTcm#&?@uL zLv$YY8f5&flO0YZ$3zSG%+M*Tg-J*j&Dt=nPt|FvE&I6oQP$EQZ&SdztAzc)I^gC# zCXJd}W|lh=wF>f@g?v0ALq51FCQRxK=-YS0jC? zpVIKVHPx3R4sMvm=ik)yRQ}=Y5kD@OX=2%cK&hX?ZE}ux93#>!^;r&bzv&k*d%%gZ z6_kX{OKiz#tic%H2Lw0I^j~QQED6M4FFkF4KXy)0#ZRMxir-vC(&t8m= z@tDc06K{(iCk*7_y>Z=`r_~c39PNllk_Zd)-4~9X(-}EtD5MqMwUH=!2PAjvna>>PUuiS!rHsyEM%+U>pK1KNOE2AOI=n5`d9=(Z&P_T%* zwGS3yX%bR1|HhU+RBed?<7vMtd%u=OJv=G{+wWq?NN8dris7H@w^hT_nnz66gH?>} z+8$JXl{r*!>%X0k8usaGCX>0H|7zI54Tej*;E&1uCEPV(*!ClIXyj(@C(cuM8N-7^ zKf0Zw;LetHshCXFu(`b*!&RhXZe|SDYVEW2_>*-#>lL7;QRJ z7}2u&$#GX=!$lR=!h;u5}*0X+qUA!LcD6VN$%LiNMnj#>1P! z%N>9PA96&gJhq8OS!q(MU4mV0!ZHTF#1+Cj`wouCjz=+CCC^KEUEv!a)2s>29`F@l zYOc!3M(FNzrf*qRFOVW!KXRYSVBP3ct=&fA;Ur5L-h^zi$-TZ*-;h%XQ&jJvh37Ex zsBvSp)zYf6X}p2gLNewhJ{}KT-D_rIV4cm6XZiuX7}e+ytn_6kM9Vq<0)s5SN&KTa z%dDBxx*q}JHDD?M78D%mD2d23k;}a&P-GmQn-?Dbw*FnW-wZLV=;!B{D-J`qE&3R5 zOyExYlhyWRosuH8TRe#~7+qA9vPoRs)vS%R{8uusZb_w9a@=<+qMO@Vk4;?Br7epJ zU(BRLQ$w0#t2ZElj;r=gS0XvGY-U`CPq9D5PO7$;~V=85%~RQP9aK0VcH z>~ivsyGrT$`#p(#bB}1I$4sIbJU+D6e&+ys=LhZZNpDwkAIY_~`30he( zsI6K7eG!*Y+^5+{$sqRL%!v-!vz7%BJu}hijT@)s2}rtC%;b&9`mmxjT9}Y3D!HFW zs)BKYN2-Q#jwjGspdhigLbr)F87DHK%0F7IwQ{euf>9^2H&-Mu3SrBe0yG|gfXuoy z>A6=)HDc>bNr63RJ_vH(pA?9n>7lJ&v zcrI`;kYXxU`L2Bfg-4fC+ZESmu<+7<(9xX$Q=TBCBGjF@gmvgV*7(3gU}khFze@45 z4CXR&m;KCVyTzjj#|Wf*sdQJ@Xu48a>XVt%5uN8YDPp6k4IkqKc0bli0Jup=fr_vi zUW0p}atKJ;A)6;n8pOCj-3Z!0$S{*@ z*6IY|^AtDfNh>1HX% zyNI9nG#Foi>B#w)Y_TQiz|bhBUWUOwPZlB|?mTCg#naiOj9#1TTIIXG_aUrrJ)PRM z-={DVHfCCUb53$-IP;CIqHYAe!l1*&t}7HbmOkEn-ddvhEIg#=4F?jXC6*t=Su^G5a{p$!2EFjC~$B-`^t2VRRoXQ3h^pK$>+|v{!ZX~Z_3MKI{w1?S&gSwSr?iD5=#>-C3O0n}N8a zlg}pwZ!=oN9#_hh*sLb#@c+6dFItizpB-2 zRHa|)Q{OYL-;Joz0`KHQFaU7K^504>OD}{m_J067FXRWpVsjS}_;jRzCqeen>?<^72TgYqTs~Cq3Cmvmy`o1mBCLOnn)&of!MOcaO|icZff- z^sIK$+HHz>OTq!pP$@}=Y{QXIRk=vQS<{BE66aE>m=ieI$Pdq zX_Pu|g)An4X>xq;HawZHlcUTf8FTg~!voq8R>`{1X51g}zK~u#QHJA7;Ntw^J06=u zprPupb3v(?iT07PHj`Ly#LlEyhRK_X68odd6nx~Ga^@xsPxT2J(B)%5;zCV=`2ihv z)+>-?yt1vjA~qW3Lfa<7DqC*6(HL^;?UaCB4KAuLYy6r$Zkx}5*GZ9P2@^}0kdt^H z;q^{mc1+DYU+_KZ9<&6Gp<#hSL~Ueeq$Jzt4i!_RvSqwDvsryEm%?@Ss5Xh zNGuVfvxO9{c78M$->}UdGBF+MbNHqpFY!{Ddp7;%Nv^&4R{DZ=T&7$;y^*ggkS66D zDfMBq&_a7T(rW`2PB-l`}N> zY%EzMPP1^LV3Sn-!SRO~HnuOV$X>Q(@B(x+_b2*3OoA^x=YzbWgbzlbkWwS`u`s_w zDo>AG?O`Q>FL;-%RbgkpQy6&A-P)*142_R91m)}B3x}x+HO_=&Tt3~|!5OkCs^V|z zr&HObG=~^!^F0qJL*)AR^O~NL6rO>9!+vlANW`DF`D(QPf&Fj(_4mN(GdcgoN}n;y zL(bW$?r+|P-C~xf_^tfm3s^}OdG6nu(biD}{@hjIJS@lGA$|)#NU+qToODP%Acz_8 zH{-=Nuv`REVTW0uT)*A_-&6f*sc8P?u8E~!<@{@le|%Q<5dlS|h!7AA zB8YSp1d(3k2Yze4&-dn^JLj%BbMD;PXV%_z(t8&e%&wA=vw^6osX^VEn@tQHBxI*g zviht0=vDQAg9f+Q`n{d0>6 zdXlmOEstV(n(unbh|)27TYkKdz}+ow=H3DV`XGpqo)fL?+@-@2J%BdFj6)t`anPOavK`6-1Ltbc> zq&IHQdXMO4GgLggq}$N_6JmCiidwFvg=rN80>zMkK>zv=hNmylR-{m1z~q%I`1vpP zjP;bctdV|0%arm|vg`+9qzHu_ypj;g9Tsz>YBExpu+!!5>3L=Pr%LgUdsm3@O;4>@ z;pJQu4{wQAUN;^79?XX_cjmQ>U#N@c{8O5*tiAPizI9}G1U;M7Y1b6o{>6Iq;?`~N@K zD+q8ugDjQ61|_be-gTN1o3y~=ZeXPAa)p-0jztSi!i!S3I9dXt{fXmMeLg$n(HFXc z1)G1t{Q8_2(nG4pvw;mi_m>l{f-5(+B|Ykr12wR3uW12zBDu*y(p+O_++9n#w52^ zX4zalQtQnm&DH4R=LYj1adJ9Oh729c@f)#7c}`TbFO9miBTRV`YA3s)Us)bc6%;z7 zsc}n+r5_8LeqWLP7A$r@GCqGsC{D6`#(S^Z_V(*uW0PdcL}VRk@p^74*aBff`-Bm0 z0JOAb(%L!uC(3KrW|52bn@Dh-i<+iq&kN={h&G2O*g!ex>d7tz@Xr*;q!5$~HD;Ej z;WsOzMT+#_K@Z$FaFFQ8l=}MNsp9*(VQs^v6i%MST2yym+@?&=oG7*5PPCYaZ75V9-SQ0K`b-yo4#_c9P0v$-#13>(dIA16$kEi(_3`z{4@e z2wqFoF84m0j7FjB()tWVw*0EgX@oRRH^GhTw8l3bvTsu)sRfW~N(;p(e+VbRw&7OM znxjOmAF=$yZZ(}W_j&ITwuBIuFFN}QEEr@H?JZj?k6fe$J~=8YXJhEQ;71ynx^5<{ z3eBu3IV*{l)1Mv@>ea?nF5dt45^#7_;J*>ulSU;Mm^{&nr-(y9 zY-yq!Rps&2*5}P(<>I{hJ0&40rBIG;+5~Y+ueA7|L?2# zzHt0;yGkTgeOX%oEO5@BVaS<^Vy9gf)cI*$_lfK0<9K=FNYzx%{?EOwuiRBymA$3m z<}-=zifaU(00XkvTk1Dz?b3bl6k}S3FXyZWLO-|`_zis+4g2g0I|#x13b+*vTY;$Tedf##rI@_<& zlM~7-$r>LnIv)^IA;{NO^HphFM90HE&5nm@>Ja~3ihjp_ay`s`sU6%Gt?4>wF}drz zn+xn9T7343Mns$NxU6u>>y}h zvnD$e8}d9_`}FgxUaQy&w9aK*xAv9f(~$x&0pHc0jNXks--c|Rsb(>rP~{$H020c= z+sBkdW06AwLoy)~(hih)Lvc71%5izimeH^c*3caY6JN^ii}Y}`ey2n3B(j39=Mb}3Eb$)Qy8V?c_+`vrjyh~|* z-g%b;2?}2apROR}q{QPi4FJ@>dz#?9fDRYvwss{wSyx~(y8!eO+)xfgpQJCxqJc=x z5+vbC(${3s2u{utB9ThcH)qiZNzM{RUJ*<%fDY-qXQ~dB{1~yWV~icIg<%|bPkxMP zUclj=yflG~54S&zBn5E`f}!zfkxDYf4L&4@#)wwGokRZeCM*uvvg`p5_i>Yw$X}L$ zYVwj=ZIU2C8PRYaWYB44S+f+RB$r#@P$(5vq>?T;=#GX4V?b%rXMQk|p>US&DBSdf zOz-5gQKaZHAK3@hc!3UAq9(X6;vJK+o=_|i4jzmsW%GC)@Z)aAdu^1m)F9r4XOonRt0QeLiAWZ)7!n!+j^RH&2m?+R2ldI=WN` zx{;8hXb6QOnKGkNxkxjwXhStJmMZgO9^%rCB^!S_2%t1t6v#=PW(pptMI)p5SOk75 z-(4Pi(IDSvsWh;<+G{80ljwN(b#N{=h|G4;$QlBaXlo+uZ+@1o*(-1T z%I^A@P6E;z&NKNF_2nC6yM5uECr}6XcoaPK(CK)K>dPIoHi^dK>|*gX2B`$I=BS2q zNQ{DL1vfh6w*!FU^Ii1OE)mXcm9Hoo2;1yR4K#yEOLhVg&&I@Fhow#A<6a0h88_HU z6?+4ep-qPMG>xonWrZ{~LJNiv{_1-&F6X8&Ewv&+CPLo+#Aav26^<0{?FA3V>~GCs z??q%)@>0=h-yK~UpAK}#y7SAw_sj5>rp1fo2#U@QRK2&XeR3S5-@ahS*70Ta#;6yx z{4v+p`;G*mCZI?X zOHb2B+i@RnpP4b!n!q~3+A`JIF#kpjp)I2Tfqt|9cbGeZMS(8?r>y18`Ys#CE(5>! zRSRKn8icaQZyNn)j{Fxd(|I{_CDyKu8nu54c_O2%%~@kRiV&8}EL``5K!Bwl2YdA# zipV(Kd3GnCH(pPeXAZwT_%&xSDLjeI*M$%4A=%lJs&>e=+8txAQ4?5Ff}1`(#bX*m z9aa^@^0};)f`I+OXSL#Xl8*+>Im5_Sq4IHfsja?S(KWeJOXntRc>{9`B5N{GOX@0( zC&%5xHgp*c9Ir%d%RQo6pV{kZ`~rRIXR?@N5OY{w^C{bXd6I&eB<9UlB9lfEm|q*Z z9mdK#YoQ{q)I_PX>z8QKqBAZTGT4lbdJ|=OB%_ zN|4r+Vp`FS=bz}tQTNd2z4a?DZMd7EN{6ZGC}Q>yPjn3wB$l6P% zDk<26IcL81N4ric>>UQH)alZkIyHVnyt)FRZMk2v8h8Y3V&qG4@)u+Dwp)SD!@)CWg)P2IT_qu9 z1N|dd4ILG+5eKJoA&3kFV*2NpGlv#^!p;XdOZGRbV)$2#Ci)5o&!3pTFDuu-Vl>bZ z9HM_>{x0#9p#5c{=nf7+=-F9L%FnOKo_>!lIS6!C?EjlQD;mklj%McsQ&#-h{Xg{! B4AuYu diff --git a/stages/tests.py b/stages/tests.py index 1c03dee..d7ddb0e 100644 --- a/stages/tests.py +++ b/stages/tests.py @@ -3,7 +3,7 @@ import os from datetime import date from django.contrib.auth.models import User -from django.test import TestCase +from django.test import TestCase, override_settings from django.urls import reverse from django.utils.html import escape @@ -180,7 +180,8 @@ class ImportTests(TestCase): self.client.login(username='me', password='mepassword') with open(path, 'rb') as fh: response = self.client.post(reverse('import-students'), {'upload': fh}, follow=True) - self.assertContains(response, escape("La classe '1ASEFEa' n'existe pas encore")) + msg = "\n".join(str(m) for m in response.context['messages']) + self.assertIn("La classe '1ASEFEa' n'existe pas encore", msg) lev1 = Level.objects.create(name='1') Klass.objects.create( @@ -193,9 +194,10 @@ class ImportTests(TestCase): section=Section.objects.create(name='EDE'), level=lev1, ) - with open(path, 'rb') as fh: + with open(path, 'rb') as fh: # , override_settings(DEBUG=True): response = self.client.post(reverse('import-students'), {'upload': fh}, follow=True) - self.assertContains(response, "Created objects: 2") + msg = "\n".join(str(m) for m in response.context['messages']) + self.assertIn("Created objects: 2", msg) def test_import_hp(self): teacher = Teacher.objects.create( diff --git a/stages/utils.py b/stages/utils.py index 7e39332..01fb4b2 100644 --- a/stages/utils.py +++ b/stages/utils.py @@ -12,3 +12,11 @@ def school_year(date, as_tuple=False): return (start_year, start_year + 1) else: return "%d — %d" % (start_year, start_year + 1) + + +def is_int(s): + try: + int(s) + return True + except ValueError: + return False diff --git a/stages/views.py b/stages/views.py index 9f495d6..082023e 100644 --- a/stages/views.py +++ b/stages/views.py @@ -19,6 +19,7 @@ from .models import ( Klass, Section, Student, Teacher, Corporation, CorpContact, Course, Period, Training, Availability, ) +from .utils import is_int def school_year_start(): @@ -319,33 +320,77 @@ class StudentImportView(ImportViewBase): corporation_mapping = settings.CORPORATION_IMPORT_MAPPING instructor_mapping = settings.INSTRUCTOR_IMPORT_MAPPING + def strip(val): + return val.strip() if isinstance(val, str) else val + obj_created = obj_modified = 0 + seen_students_ids = set() for line in up_file: student_defaults = { - val: line[key] for key, val in student_mapping.items() if val != 'ext_id' - } - corporation_defaults = { - val: line[key] for key, val in corporation_mapping.items() - } - instructor_defaults = { - val: line[key] for key, val in instructor_mapping.items() + val: strip(line[key]) for key, val in student_mapping.items() } + if student_defaults['ext_id'] in seen_students_ids: + # Second line for student, ignore it + continue + seen_students_ids.add(student_defaults['ext_id']) + if isinstance(student_defaults['birth_date'], str): + student_defaults['birth_date'] = datetime.strptime(student_defaults['birth_date'], '%d.%m.%Y').date() - defaults = Student.prepare_import( - student_defaults, corporation_defaults, instructor_defaults - ) - obj, created = Student.objects.get_or_create( - ext_id=line[student_rev_mapping['ext_id']], defaults=defaults) - if not created: + corporation_defaults = { + val: strip(line[key]) for key, val in corporation_mapping.items() + } + student_defaults['corporation'] = self.get_corporation(corporation_defaults) + + instructor_defaults = { + val: strip(line[key]) for key, val in instructor_mapping.items() + } + student_defaults['instructor'] = self.get_instructor(student_defaults['corporation'], instructor_defaults) + + defaults = Student.prepare_import(student_defaults) + try: + student = Student.objects.get(ext_id=student_defaults['ext_id']) + modified = False for key, val in defaults.items(): - setattr(obj, key, val) - obj.save() - obj_modified += 1 - else: + if getattr(student, key) != val: + setattr(student, key, val) + modified = True + if modified: + student.save() + obj_modified += 1 + except Student.DoesNotExist: + student = Student.objects.create(**defaults) obj_created += 1 #FIXME: implement arch_staled return obj_created, obj_modified + def get_corporation(self, corp_values): + if corp_values['ext_id'] == '': + return None + if 'city' in corp_values and is_int(corp_values['city'][:4]): + corp_values['pcode'], _, corp_values['city'] = corp_values['city'].partition(' ') + corp, created = Corporation.objects.get_or_create( + ext_id=corp_values['ext_id'], + defaults=corp_values + ) + return corp + + def get_instructor(self, corp, inst_values): + if inst_values['ext_id'] == '': + return None + try: + inst = CorpContact.objects.get(ext_id=inst_values['ext_id']) + except CorpContact.DoesNotExist: + try: + inst = corp.corpcontact_set.get( + first_name__iexact=inst_values['first_name'], last_name__iexact=inst_values['last_name'] + ) + inst.ext_id = inst_values['ext_id'] + inst.save() + except CorpContact.DoesNotExist: + inst_values['corporation'] = corp + inst = CorpContact.objects.create(**inst_values) + return inst + class HPImportView(ImportViewBase): """