management/commands/_import_utils.py

-#!/usr/bin/env python3
+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import json
 import time
@@ -44,14 +45,14 @@ class ImportCommand(BaseCommand):
         n = str(n)
         total = str(total)
         n.rjust(len(total))
-        print(f"\r ({n}/{total}) {content:10.10}", end="")
+        print(f"\r ({n}/{total}) {content:16.16}", end="")
 
     def create_parser(self, prog_name, subcommand, **kwargs):
         parser = super().create_parser(prog_name, subcommand, **kwargs)
         parser.add_argument('--nk15db', action='store', default='nk15', help='NK15 database name')
         parser.add_argument('--nk15user', action='store', default='nk15_user', help='NK15 database owner')
-        parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
-        parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
+        parser.add_argument('-s', '--save', default='map.json', action='store', help="save mapping of idbde")
+        parser.add_argument('-m', '--map', default='map.json', action='store', help="import mapping of idbde")
         parser.add_argument('-c', '--chunk', type=int, default=100, help="chunk size for bulk_create")
         return parser
 

management/commands/anonymize_data.py

+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    """
+    Command to protect sensitive data during the beta phase, to prevent a right escalation.
+    Phone number, email address, postal address, first and last name are removed.
+    """
+    def handle(self, *args, **kwargs):
+        cur = connection.cursor()
+        cur.execute("UPDATE member_profile SET "
+                    "phone_number = '0123456789', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+        cur.execute("UPDATE auth_user SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme', "
+                    "email = 'anonymous@example.com';")
+        cur.execute("UPDATE member_club SET "
+                    "email = 'anonymous@example.com';")
+        cur.close()

management/commands/import_account.py

-#!/usr/bin/env python3
+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import psycopg2 as pg
 import psycopg2.extras as pge
@@ -18,7 +19,7 @@ from member.models import Club, Profile
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
 M_DURATION = 396
-M_START = datetime.date(2019, 8, 31)
+M_START = datetime.date(2019, 8, 1)
 M_END = datetime.date(2020, 9, 30)
 
 MAP_IDBDE = {
@@ -59,6 +60,13 @@ class Command(ImportCommand):
         Every Model has to be manually created, and no magic `.save()`
         function is being called.
         """
+        # Get promotion and date of creation of the account
+        cur.execute("SELECT idbde, MIN(date) AS created_at, MIN(annee) AS promo FROM adhesions"
+                    " GROUP BY idbde ORDER BY promo, created_at;")
+        MAP_IDBDE_PROMOTION = {}
+        for row in cur:
+            MAP_IDBDE_PROMOTION[row["idbde"]] = row
+
         cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
         pk_club = 3
         pk_user = 1
@@ -82,8 +90,7 @@ class Command(ImportCommand):
                 "balance": row['solde'],
                 "last_negative": None,
                 "is_active": True,
-                "display_image": "",
-                "created_at": now()
+                "display_image": "pic/default.png",
             }
             if row["last_negatif"] is not None:
                 note_dict["last_negative"] = make_aware(row["last_negatif"])
@@ -94,12 +101,17 @@ class Command(ImportCommand):
                 else:
                     passwd_nk15 = ''
 
+                if row["idbde"] not in MAP_IDBDE_PROMOTION:
+                    # NK12 bug. Applying default values
+                    MAP_IDBDE_PROMOTION[row["idbde"]] = {"promo": 2014,
+                                                         "created_at": datetime.datetime(2014, 9, 1, 0, 0, 0)}
+
                 obj_dict = {
                     "pk": pk_user,
                     "username": row["pseudo"],
                     "password": passwd_nk15,
-                    "first_name": row["nom"],
-                    "last_name": row["prenom"],
+                    "first_name": row["prenom"],
+                    "last_name": row["nom"],
                     "email": row["mail"],
                     "is_active": True,  # temporary
                 }
@@ -111,7 +123,9 @@ class Command(ImportCommand):
                     "paid": row['normalien'],
                     "registration_valid": True,
                     "email_confirmed": True,
+                    "promotion": MAP_IDBDE_PROMOTION[row["idbde"]]["promo"],
                 }
+                note_dict["created_at"] = make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"])
                 note_dict["polymorphic_ctype"] = note_user_type
                 note_user_dict = {
                     "pk": pk_note,
@@ -137,6 +151,7 @@ class Command(ImportCommand):
                     "pk": pk_club,
                     "name": row["pseudo"],
                     "email": row["mail"],
+                    "parent_club": 1,   # All clubs depends on BDE by default
                     "membership_duration": M_DURATION,
                     "membership_start": M_START,
                     "membership_end": M_END,
@@ -145,7 +160,7 @@ class Command(ImportCommand):
                 }
                 note_club_dict = {
                     "pk": pk_note,
-                    "club_id": pk_club,
+                    "club_id": pk_club
                 }
                 alias_dict = {
                     "pk": pk_note,
@@ -153,16 +168,18 @@ class Command(ImportCommand):
                     "normalized_name": Alias.normalize(pseudo),
                     "note_id": pk_note
                 }
+                note_dict["created_at"] = make_aware(row["previous_report_date"])  # Not perfect, but good approximation
                 note_dict["polymorphic_ctype"] = note_club_type
-                bulk_mgr.add(Club(**obj_dict),
-                             Note(**note_dict),
-                             NoteClub(**note_club_dict),
-                             Alias(**alias_dict))
+                #bulk_mgr.add(Club(**obj_dict),
+                #             Note(**note_dict),
+                #             NoteClub(**note_club_dict),
+                #             Alias(**alias_dict))
                 pk_club += 1
             # row import completed
             MAP_IDBDE[row["idbde"]] = pk_note
             pk_note += 1
         bulk_mgr.done()
+        exit(0)
 
     @timed
     def import_alias(self, cur, chunk_size):

management/commands/import_activities.py

-#!/usr/bin/env python3
+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import psycopg2 as pg
 import psycopg2.extras as pge
@@ -10,18 +11,22 @@ from django.db import transaction
 
 from activity.models import ActivityType, Activity, Guest, Entry
 from member.models import Club
-from note.models import Note
+from note.models import Note, NoteUser
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
 MAP_ACTIVITY = dict()
 
 CLUB_RELOU = [
-    0, # BDE
+    0,    # BDE
     4771, # Kataclist
     5162, # Assurance BDE ?!
-    5164, #S & L
-    625, #Aspique
-    5154, #Frekens
+    5164, # S & L
+    625,  # Aspique
+    5154, # Frekens
+    3944, # DiskJok[ENS]
+    5153, # Monopo[list]
+    2351, # JdRM
+    2365, # Pot Vieux
 ]
 
 class Command(ImportCommand):
@@ -40,11 +45,13 @@ class Command(ImportCommand):
         pk_activity = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["titre"])
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             note = self.MAP_IDBDE[row["responsable"]]
             if note == 6244:
                  # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
                 note = Note.objects.get(pk=self.MAP_IDBDE[6524])
-                note = note.user_id
+                note = note.id
             organizer = Club.objects.filter(name=row["signature"])
             if organizer.exists():
                 # Try to find the club that organizes the activity.
@@ -57,7 +64,7 @@ class Command(ImportCommand):
                 "name": row["titre"],
                 "description": row["description"],
                 "activity_type_id": activity_type_id,  # By default Pot
-                "creater_id": note,
+                "creater_id": NoteUser.objects.get(pk=note).user.id,
                 "organizer_id": organizer.pk,
                 "attendees_club_id": kfet.pk,  # Maybe fix manually
                 "date_start": make_aware(row["debut"]),

management/commands/import_nk15.py

-#!/usr/env/bin python3
+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 import subprocess
 
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
-class Command(BaseCommand):
+from ._import_utils import ImportCommand
+
+class Command(ImportCommand):
     """
     Command for importing the database of NK15.
     Need to be run by a user with a registered role in postgres for the database nk15. 
@@ -12,7 +16,22 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         subprocess.call("./apps/scripts/shell/tabularasa")
-        call_command('import_account', alias=True, chunk=1000, save = "map.json")
-        call_command('import_activities', chunk=100, map="map.json")
-        call_command('import_transaction', chunk=10000, buttons=True, map="map.json")
-#
+
+        kwargs["alias"] = True
+        kwargs["chunk"] = 1000
+        kwargs["save"] = "map.json"
+        call_command('import_account', **kwargs)
+
+        del kwargs["alias"]
+        del kwargs["save"]
+        kwargs["chunk"] = 100
+        kwargs["map"] = "map.json"
+        call_command('import_activities', **kwargs)
+
+        kwargs["chunk"] = 10000
+        kwargs["map"] = "map.json"
+        kwargs["buttons"] = True
+        call_command('import_transaction', **kwargs)
+
+        call_command('make_su','-sS', 'Coq', 'erdnaxe', 'Krokmou', 'PAC', 'Pollion', 'TLinux', 'ÿnérant')
+        call_command('syncsql')

management/commands/import_transaction.py

-#!/usr/bin/env python3
+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 import re
 import psycopg2 as pg
 import psycopg2.extras as pge
@@ -8,28 +10,46 @@ import copy
 
 from django.utils.timezone import make_aware
 from django.db import transaction
+from django.contrib.contenttypes.models import ContentType
 
 from note.models import (TemplateCategory,
                          TransactionTemplate,
                          Transaction,
                          RecurrentTransaction,
-                         SpecialTransaction
+                         SpecialTransaction,
+                         MembershipTransaction,
                          )
 from note.models import Note, NoteClub
-from activity.models import Guest, GuestTransaction
+from activity.models import Guest, GuestTransaction, Entry
 
-from member.models import Membership, MembershipTransaction
+from member.models import Membership
+from treasury.models import Remittance, SpecialTransactionProxy
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
+MAP_TRANSACTION = dict()
+MAP_REMITTANCE = dict()
+
+# from member/fixtures/initial
 BDE_PK = 1
 KFET_PK = 2
+
+# from note/fixtures/initial
 NOTE_SPECIAL_CODE = {
     "espèce": 1,
     "carte": 2,
     "chèque": 3,
     "virement": 4,
 }
+# from permission/fixtures/initial
+BDE_ROLE_PK = 1
+KFET_ROLE_PK = 2
 
+CT = {
+    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
+    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
+    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
+    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
+}
 
 def get_date_end(date_start):
     date_end = copy.deepcopy(date_start)
@@ -47,9 +67,11 @@ class Command(ImportCommand):
     def add_arguments(self, parser):
         parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
+        parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
+                                                                                       "don't save them")
 
     @timed
-    def import_buttons(self, cur, chunk_size):
+    def import_buttons(self, cur, chunk_size, import_buttons):
         self.categories = dict()
         self.buttons = dict()
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
@@ -58,7 +80,7 @@ class Command(ImportCommand):
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["label"])
             if row["categorie"] not in self.categories:
-                cat = TemplateCategory.objects.create(name=row["categorie"])
+                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
                 cat.save()
                 self.categories[row["categorie"]] = cat.pk
             obj_dict = {
@@ -72,28 +94,31 @@ class Command(ImportCommand):
             }
             if row["label"] in self.buttons:
                 obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
-            bulk_mgr.add(TransactionTemplate(**obj_dict))
+            if import_buttons:
+                bulk_mgr.add(TransactionTemplate(**obj_dict))
             self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
         bulk_mgr.done()
 
     def _basic_transaction(self, row, obj_dict, child_dict):
         if len(row["description"]) > 255:
-            obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
+            obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
         return obj_dict, None, None
 
     def _template_transaction(self, row, obj_dict, child_dict):
-        if self.categories.get(row["categorie"]):
-            child_dict["category_id"] = self.categories[row["categorie"]]
-        elif "WEI" in row["description"]:
-            return obj_dict, None, None
-        elif self.buttons.get(row["description"]):
+        if self.buttons.get(row["description"]):
             child_dict["category_id"] = self.buttons[row["description"]][1]
             child_dict["template_id"] = self.buttons[row["description"]][0]
+        # elif self.categories.get(row["categorie"]):
+        #     child_dict["category_id"] = self.categories[row["categorie"]]
+        elif "WEI" in row["description"]:
+            return obj_dict, None, None
         else:
             return obj_dict, None, None
+        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
         return obj_dict, child_dict, RecurrentTransaction
 
     def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
+        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
         obj_dict2 = obj_dict.copy()
         child_dict2 = child_dict.copy()
         child_dict2["membership_id"] = pk_membership
@@ -103,6 +128,7 @@ class Command(ImportCommand):
     def _special_transaction(self, row, obj_dict, child_dict):
         # Some transaction uses BDE (idbde=0) as source or destination,
         # lets fix that.
+        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
         field_id = "source_id" if row["type"] == "crédit" else "destination_id"
         if "espèce" in row["description"]:
             obj_dict[field_id] = 1
@@ -125,25 +151,30 @@ class Command(ImportCommand):
         return obj_dict, child_dict, SpecialTransaction
 
     def _guest_transaction(self, row, obj_dict, child_dict):
-        # Currently GuestTransaction is related to a Guest.
-        # This is not ideal and should be change to the Entry of this Guest.
+        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
         m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
         if m:
+            activity_name = m.group(1)
             first_name, last_name = m.group(2), m.group(3)
-            guest_id = Guest.objects.filter(first_name__iexact=first_name,
-                                           last_name__iexact=last_name).first().pk
-            child_dict["guest_id"] = guest_id
+            if first_name == "Marion" and last_name == "Bizu Pose":
+                first_name, last_name = "Marion Bizu", "Pose"
+            entry_id = Entry.objects.filter(
+                    activity__name__iexact=activity_name,
+                    guest__first_name__iexact=first_name,
+                    guest__last_name__iexact=last_name,
+            ).first().pk
+            child_dict["entry_id"] = entry_id
         else:
-            raise(f"Guest not Found {row['id']} {first_name}, last_name")
+            raise Exception(f"Guest not Found {row['id']} first_name, last_name")
 
         return obj_dict, child_dict, GuestTransaction
 
     @timed
     @transaction.atomic
-    def import_transaction(self, cur, chunk_size, idmin):
+    def import_transaction(self, cur, chunk_size, idmin, save=True):
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute(
-            f"SELECT t.date AS transac_date, t.type, t.emetteur,\
+            f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
               t.destinataire,t.quantite, t.montant, t.description,\
               t.valide, t.cantinvalidate, t.categorie, \
               a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
@@ -154,8 +185,21 @@ class Command(ImportCommand):
         n = cur.rowcount
         pk_membership = 1
         pk_transaction = 1
+        kfet_balance = 0
         for idx, row in enumerate(cur):
-            self.update_line(idx, n, row["description"])
+            if save or idx % chunk_size == 0:
+                self.update_line(idx, n, row["description"])
+
+            MAP_TRANSACTION[row["id"]] = pk_transaction
+
+            if not save:
+                pk_transaction += 1
+                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
+                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
+                    if not isinstance(note, NoteClub):
+                        pk_transaction += 1
+                continue
+
             try:
                 date = make_aware(row["transac_date"])
             except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
@@ -165,35 +209,45 @@ class Command(ImportCommand):
             obj_dict = {
                 "pk": pk_transaction,
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
+                "polymorphic_ctype": None,
                 "source_id": self.MAP_IDBDE[row["emetteur"]],
-                "created_at": date,
                 "amount": row["montant"],
+                "created_at": date,
+                "destination_alias": "",
+                "invalidity_reason": None,
                 "quantity": row["quantite"],
                 "reason": row["description"],
+                "source_alias": "",
                 "valid": row["valide"],
             }
+            if len(obj_dict["reason"]) > 255:
+                obj_dict["reason"] = obj_dict["reason"][:252] + "..."
             # for child transaction Models
-            child_dict = {"pk": obj_dict["pk"]}
+            child_dict = {"pk": pk_transaction}
             ttype = row["type"]
+            # Membership transaction detection and import
             if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                 note = Note.objects.get(pk=obj_dict["source_id"])
                 if isinstance(note, NoteClub):
-                    child_transaction = None
+                    child_transaction = None    # don't bother register clubs
                 else:
                     user_id = note.user_id
                     montant = obj_dict["amount"]
-                    obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    (obj_dict0,
+                     child_dict0,
+                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                     bde_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": KFET_PK,
+                        "club_id": BDE_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": min(500, montant)
                     }
                     pk_membership += 1
                     pk_transaction += 1
-                    obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    obj_dict, child_dict, child_transaction =\
+                        self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                     # Kfet membership
                     # BDE Membership
                     obj_dict["pk"] = pk_transaction
@@ -201,23 +255,24 @@ class Command(ImportCommand):
                     kfet_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": BDE_PK,
+                        "club_id": KFET_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": max(montant - 500, 0),
                     }
                     obj_dict0["amount"] = bde_dict["fee"]
                     obj_dict["amount"] = kfet_dict["fee"]
+                    kfet_balance += kfet_dict["fee"]
                     # BDE membership Transaction is inserted before the Kfet membershipTransaction
                     pk_membership += 1
                     pk_transaction += 1
                     bulk_mgr.add(
+                        Membership(**bde_dict),
+                        Membership(**kfet_dict),
                         Transaction(**obj_dict0),
                         child_transaction(**child_dict0),
                         Transaction(**obj_dict),
                         child_transaction(**child_dict),
-                        Membership(**bde_dict),
-                        Membership(**kfet_dict),
                     )
                     continue
             elif ttype == "bouton":
@@ -226,16 +281,91 @@ class Command(ImportCommand):
                 obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
             elif ttype == "invitation":
                 obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
-            if ttype == "don" or ttype == "transfert":
+            elif ttype == "don" or ttype == "transfert":
                 obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
             else:
                 child_transaction = None
                 # create base transaction object and typed one
             bulk_mgr.add(Transaction(**obj_dict))
             if child_transaction is not None:
+                child_dict.update(obj_dict)
                 bulk_mgr.add(child_transaction(**child_dict))
             pk_transaction += 1
         bulk_mgr.done()
+
+        # Update the balance of the BDE and the Kfet club
+        note_bde = NoteClub.objects.get(pk=5)
+        note_kfet = NoteClub.objects.get(pk=6)
+        note_bde.balance -= kfet_balance
+        note_kfet.balance += kfet_balance
+        note_bde.save()
+        note_kfet.save()
+
+    @timed
+    def set_roles(self):
+        bulk_mgr = BulkCreateManager(chunk_size=10000)
+        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
+        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
+        n = len(bde_membership_ids)
+        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
+            self.update_line(idx, n, str(idx))
+            bulk_mgr.add(
+                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
+                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
+            )
+            bulk_mgr.done()
+
+    @timed
+    @transaction.atomic
+    def import_remittances(self, cur, chunk_size):
+        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
+        cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
+        n = cur.rowcount
+        pk_remittance = 1
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["commentaire"])
+
+            MAP_REMITTANCE[row["id"]] = pk_remittance
+
+            remittance_dict = {
+                "pk": pk_remittance,
+                "date": make_aware(row["date"]),
+                "remittance_type_id": 1,    # Only Bank checks are supported in NK15
+                "comment": row["commentaire"],
+                "closed": row["close"],
+            }
+            bulk_mgr.add(Remittance(**remittance_dict))
+
+            pk_remittance += 1
+
+        bulk_mgr.done()
+
+    @timed
+    def import_checks(self, cur):
+        cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
+                    "FROM cheques ORDER BY id;")
+        n = cur.rowcount
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["nom"])
+
+            if not row["idremise"]:
+                continue
+
+            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
+            tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
+            tr.save()
+
+            tr = tr.transaction
+            tr.last_name = row["nom"]
+            tr.first_name = row["prenom"]
+            tr.bank = row["banque"]
+            try:
+                tr.save()
+            except:
+                print("Failed to save row: " + str(row))
+
     @timed
     def handle(self, *args, **kwargs):
         # default args, provided by ImportCommand.
@@ -246,5 +376,9 @@ class Command(ImportCommand):
 
         if kwargs["map"]:
             self.load_map(kwargs["map"])
-        self.import_buttons(cur, kwargs["chunk"])
-        self.import_transaction(cur, kwargs["chunk"], 0)
+        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
+        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
+        if not kwargs["nosave"]:
+            self.set_roles()
+        self.import_remittances(cur, kwargs["chunk"])
+        self.import_checks(cur)

management/commands/refresh_highlighted_buttons.py

+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from django.utils import timezone
+
+from note.models import RecurrentTransaction, TransactionTemplate
+
+
+class Command(BaseCommand):
+    """
+    Command to add the ten most used buttons of the past month to the highlighted buttons.
+    """
+    def handle(self, *args, **kwargs):
+        queryset = RecurrentTransaction.objects.filter(
+            template__display=True,
+            valid=True,
+            created_at__gte=timezone.now() - timedelta(days=30),
+        ).values("template").annotate(transaction_count=Count("template")).order_by("-transaction_count")[:10]
+        for d in queryset.all():
+            button_id = d["template"]
+            button = TransactionTemplate.objects.get(pk=button_id)
+            self.stdout.write(self.style.WARNING("Highlight button {name} ({count:d} transactions)..."
+                                                 .format(name=button.name, count=d["transaction_count"])))
+            button.highlighted = True
+            button.save()

management/commands/syncsql.py

+# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from django.db import connection
+
+
+from polymorphic.models import PolymorphicModel
+
+NO_SEQ = [
+    "Session",
+    "Token",
+    "WEIRole",  # dirty fix
+]
+
+class Command(BaseCommand):
+    """
+    Command to synchronise primary sequence of postgres after bulk insert of django.
+    """
+
+    def add_arguments(self,parser):
+        parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
+        return parser
+
+    def handle(self, *args, **kwargs):
+        app_list = kwargs["apps"]
+        if len(app_list):
+            model_classes = list()
+            for app in app_list:
+                model_classes += apps.get_app_config(app).get_models()
+        else:
+            # no app specified, sync everything
+            model_classes = apps.get_models(include_auto_created=True)
+
+        db_names = [
+            m._meta.db_table for m in model_classes
+            if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count() > 1
+        ]
+        com = "BEGIN;\n"
+        for db_name in db_names:
+            com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1),' \
+                   f' max("id") IS NOT null) FROM "{db_name}";\n'
+        com += "COMMIT;"
+        print(com)
+        cur = connection.cursor()
+        cur.execute(com)
+        cur.close()

shell/tabularasa

 #!/usr/bin/sh
+sudo service postgresql stop
+sudo service postgresql start
 sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
 echo 'reset db';
 find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete