Comparer les révisions
50 Révisions
master
...
fusion_abp
Auteur | SHA1 | Date |
---|---|---|
Vincent Adolphe | d82ec5d77f | |
Vincent Adolphe | accd2affbd | |
Vincent Adolphe | 092d7dae9e | |
Vincent Adolphe | 823daa8530 | |
Vincent Adolphe | 08162c76c7 | |
Vincent Adolphe | b6a6826940 | |
Vincent Adolphe | b3c6fad556 | |
Vincent Adolphe | f7bdf0e1cc | |
Vincent Adolphe | 2979294f83 | |
Vincent Adolphe | 3d06ed3a39 | |
Vincent Adolphe | e6aa1ca413 | |
Vincent Adolphe | b9a5a3622d | |
Vincent Adolphe | 9b1784d558 | |
Vincent Adolphe | a4fe772028 | |
Vincent Adolphe | 07eae89642 | |
Vincent Adolphe | 769f416c49 | |
Vincent Adolphe | 32472e7a07 | |
Vincent Adolphe | 72d438e2d5 | |
Vincent Adolphe | 7d201b2d66 | |
Vincent Adolphe | a2a03fef8a | |
Vincent Adolphe | f4a9780f5b | |
Vincent Adolphe | f6ee401d6f | |
Vincent Adolphe | 47a6e211b0 | |
Vincent Adolphe | 0fe4b4eddf | |
Vincent Adolphe | 9a2819b5ee | |
Vincent Adolphe | ff91904ec7 | |
Vincent Adolphe | a075e6a741 | |
Vincent Adolphe | 853bdca605 | |
Vincent Adolphe | 687a2760de | |
Vincent Adolphe | 5573ec0197 | |
Vincent Adolphe | cd4f054fe8 | |
Vincent Adolphe | 858db407a8 | |
Vincent Adolphe | 91e9e92779 | |
Vincent Adolphe | 8c27c9d87d | |
Vincent Adolphe | cf1436e762 | |
Vincent Adolphe | 280b5655f8 | |
Vincent Adolphe | 6240b93b97 | |
Vincent Adolphe | 9332776f94 | |
Vincent Adolphe | 42f77d25ab | |
Vincent Adolphe | 67c459d41c | |
Vincent Adolphe | acc42275f8 | |
Vincent Adolphe | 82fbd084ab | |
Vincent Adolphe | ccec5da4f8 | |
Vincent Adolphe | b8745717ee | |
Vincent Adolphe | 1d8e9f8687 | |
Vincent Adolphe | 6c9ecff964 | |
Vincent Adolphe | ee58d96e8c | |
Vincent Adolphe | e52aeb754c | |
Vincent Adolphe | 965bfc5942 | |
Vincent Adolphe | 1c9e9b24ed |
|
@ -42,6 +42,7 @@
|
|||
#
|
||||
# /!\ Required in production.
|
||||
#DJANGO_DATABASE_URL=postgres://user:password@127.0.0.1:5432/gaby
|
||||
#DJANGO_DATABASE_URL_BIOPIC=mysql://user:password@127.0.0.1:3306/gaby
|
||||
#DJANGO_DATABASE_URL=mysql://user:password@127.0.0.1:3306/gaby
|
||||
|
||||
# Email configuration for sending messages, as an URI.
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
class BiopicRouter:
|
||||
def db_for_read(self, model, **hints):
|
||||
return 'biopic' if model._meta.app_label == 'biopic' else None
|
|
@ -0,0 +1,52 @@
|
|||
import logging
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import IntegrityError, transaction
|
||||
|
||||
from gaby.biopic import merge
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Merge ABP database into Gaby"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'--commit',
|
||||
'-c',
|
||||
dest='commit',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='actually make change (default is dryrun)',
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
logger.info(
|
||||
"Begin of operations - {}dryrun".format(
|
||||
"NOT " if options['commit'] else ""
|
||||
)
|
||||
)
|
||||
try:
|
||||
with transaction.atomic():
|
||||
merge.run()
|
||||
if not options['commit']:
|
||||
raise IntegrityError('dry_run')
|
||||
except IntegrityError as err:
|
||||
if 'dry_run' in err.args:
|
||||
self.stdout.write(
|
||||
self.style.HTTP_NOT_MODIFIED(
|
||||
"DRY RUN - no change in database"
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
self.stdout.write(
|
||||
self.style.WARNING("REAL RUN - changes committed in database")
|
||||
)
|
||||
logger.info(
|
||||
"End of operations - {}dryrun".format(
|
||||
"NOT " if options['commit'] else ""
|
||||
)
|
||||
)
|
|
@ -0,0 +1,59 @@
|
|||
from django.core.management.base import BaseCommand
|
||||
from django.urls import reverse
|
||||
|
||||
from gaby.biopic.utils import (
|
||||
BpStructureCache,
|
||||
CommuneCache,
|
||||
PersonneCache,
|
||||
StructureCache,
|
||||
)
|
||||
|
||||
GABY_PREFIX = 'https://bio-hdf.sloli.fr/gaby'
|
||||
|
||||
MODEL_MTX = { # ModelName: ModelCache, use get_admin_url()
|
||||
'Structure': (StructureCache, True),
|
||||
'BpStructure': (BpStructureCache, False),
|
||||
'Commune': (CommuneCache, True),
|
||||
'Personne': (PersonneCache, True),
|
||||
}
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Check dupes based on similary name."
|
||||
"available for models: {}".format(",".join(MODEL_MTX.keys()))
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('model', help='search dupes in this model')
|
||||
|
||||
def get_url(self, model, use_get_admin_url):
|
||||
if not use_get_admin_url:
|
||||
return model.url()
|
||||
return GABY_PREFIX + reverse(
|
||||
'admin:{0}_{1}_change'.format(
|
||||
model._meta.app_label, model.__class__.__name__.lower()
|
||||
),
|
||||
args=[model.pk],
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
cpt = 0
|
||||
sub_cpt = 0
|
||||
cache_class, use_gau = MODEL_MTX.get(options['model'], (None, None))
|
||||
if cache_class is None:
|
||||
print("Model {} inconnu".format(options['model']))
|
||||
print(
|
||||
"Les models valides sont {}".format(
|
||||
", ".join(MODEL_MTX.keys())
|
||||
)
|
||||
)
|
||||
return
|
||||
model_cache = cache_class()
|
||||
for key, val_list in model_cache.get_dupes():
|
||||
print("## {}".format(key))
|
||||
for val in val_list:
|
||||
print(" {}".format(self.get_url(val, use_gau)))
|
||||
sub_cpt += 1
|
||||
cpt += 1
|
||||
print("# {} doublons ({} lignes)".format(cpt, sub_cpt))
|
|
@ -0,0 +1,108 @@
|
|||
import logging
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import IntegrityError
|
||||
|
||||
from .models import BpStructure
|
||||
from .utils import (
|
||||
StructureCache,
|
||||
StructureFillField,
|
||||
get_gaby_struct,
|
||||
get_skip_status,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run():
|
||||
"""
|
||||
import records from biopic and merge them
|
||||
into gaby
|
||||
"""
|
||||
logger.info('attempt to merge biopic into gaby')
|
||||
# initialize a cache of slugify(gaby.structure.designation)
|
||||
gaby_str_cache = StructureCache()
|
||||
gaby_str_fill = StructureFillField()
|
||||
cpt = 0
|
||||
cpt_existant = 0
|
||||
cpt_import = 0
|
||||
cpt_import_err = 0
|
||||
for bpstruct in BpStructure.objects.not_npdc():
|
||||
skip_struct, reason = get_skip_status(bpstruct)
|
||||
if skip_struct:
|
||||
logger.info(
|
||||
"{}({}) ignoré ({})".format(bpstruct, bpstruct.id, reason)
|
||||
)
|
||||
continue
|
||||
cpt += 1
|
||||
struct, reason = get_gaby_struct(bpstruct, gaby_str_cache)
|
||||
if struct is not None:
|
||||
cpt_existant += 1
|
||||
logger.info(
|
||||
"{}({}) trouvé dans gaby({}), "
|
||||
"ajout d'un lien vers {}({})".format(
|
||||
bpstruct, bpstruct.id, reason, struct, struct.id
|
||||
)
|
||||
)
|
||||
struct.url_old_ref = bpstruct.url()
|
||||
struct.save()
|
||||
gaby_str_fill.add_tag_import_manuel(struct)
|
||||
continue
|
||||
# at this point it's a brand new structure to import from biopic
|
||||
cpt_import += 1
|
||||
struct = gaby_str_fill.build_from(bpstruct)
|
||||
struct_is_clean = True
|
||||
err_validation = None
|
||||
try:
|
||||
struct.full_clean()
|
||||
except ValidationError as err:
|
||||
cpt_import_err += 1
|
||||
err_validation = err
|
||||
struct_is_clean = False
|
||||
try:
|
||||
struct.save()
|
||||
except IntegrityError as err:
|
||||
logger.error(
|
||||
"{}({}) erreur critique d'importation: [{}]".format(
|
||||
bpstruct, bpstruct.id, err
|
||||
)
|
||||
)
|
||||
raise # raise anyway, the transaction is broken
|
||||
else:
|
||||
if err_validation is not None:
|
||||
for f_name, f_errors in err_validation.message_dict.items():
|
||||
logger.warning(
|
||||
"{} erreur de validation: {}[{}] ({})".format(
|
||||
struct.get_absolute_url(),
|
||||
f_name,
|
||||
" / ".join(f_errors),
|
||||
struct,
|
||||
)
|
||||
)
|
||||
# log it
|
||||
logger.debug(
|
||||
"{}({}) importé dans gaby: {}".format(
|
||||
bpstruct, bpstruct.id, struct.get_absolute_url()
|
||||
)
|
||||
)
|
||||
# a new structure has been successfully imported into gaby
|
||||
# override create_date with date_maj
|
||||
gaby_str_fill.reset_create_date(bpstruct, struct)
|
||||
struct.save()
|
||||
# add m2m fields
|
||||
gaby_str_fill.add_m2m(bpstruct, struct, struct_is_clean)
|
||||
# add related fields
|
||||
gaby_str_fill.add_related(bpstruct, struct)
|
||||
# update the search cache
|
||||
gaby_str_cache.add_to_cache(struct)
|
||||
# at this point related fields (M2M and FK to this struct)
|
||||
# could be imported
|
||||
# import personne physique / contact
|
||||
# import adhesion
|
||||
# import misc.
|
||||
logger.info(
|
||||
"Stats: Total: {}, Existant: {}, Importé(total): {}, "
|
||||
"Importé(avec erreur de validation): {}".format(
|
||||
cpt, cpt_existant, cpt_import, cpt_import_err
|
||||
)
|
||||
)
|
Fichier diff supprimé car celui-ci est trop grand
Voir la Diff
|
@ -0,0 +1,162 @@
|
|||
# from '2019 05 15 Doublons AbpGaby.xls'
|
||||
|
||||
STR_ABP2GABY = {
|
||||
940: 8728,
|
||||
55: 8816,
|
||||
107: 9057,
|
||||
112: 9074,
|
||||
139: 9164,
|
||||
148: 9181,
|
||||
181: 9386,
|
||||
188: 9467,
|
||||
189: 9476,
|
||||
190: 9484,
|
||||
204: 9636,
|
||||
223: 9747,
|
||||
252: 10072,
|
||||
259: 10103,
|
||||
316: 10468,
|
||||
491: 10666,
|
||||
481: 11050,
|
||||
482: 11058,
|
||||
502: 11141,
|
||||
510: 11162,
|
||||
559: 11342,
|
||||
520: 11438,
|
||||
73: 12320,
|
||||
691: 12348,
|
||||
357: 12374,
|
||||
277: 12573,
|
||||
290: 12763,
|
||||
183: 12788,
|
||||
553: 12789,
|
||||
986: 12834,
|
||||
172: 12884,
|
||||
938: 13165,
|
||||
56: 13272,
|
||||
106: 13274,
|
||||
157: 13275,
|
||||
321: 13276,
|
||||
364: 13277,
|
||||
426: 13278,
|
||||
442: 13279,
|
||||
519: 13280,
|
||||
619: 13281,
|
||||
626: 13282,
|
||||
651: 13283,
|
||||
697: 13284,
|
||||
913: 13285,
|
||||
939: 13286,
|
||||
968: 13287,
|
||||
994: 13288,
|
||||
1021: 13289,
|
||||
91: 13408,
|
||||
448: 13500,
|
||||
879: 13551,
|
||||
62: 13557,
|
||||
822: 13559,
|
||||
4: 13689,
|
||||
}
|
||||
|
||||
STR_BLACKLIST = {
|
||||
363,
|
||||
441,
|
||||
782,
|
||||
1112,
|
||||
1109,
|
||||
1018,
|
||||
360,
|
||||
416,
|
||||
652,
|
||||
952,
|
||||
878,
|
||||
1057,
|
||||
475,
|
||||
837,
|
||||
757,
|
||||
732,
|
||||
941,
|
||||
798,
|
||||
77,
|
||||
973,
|
||||
746,
|
||||
747,
|
||||
1040,
|
||||
}
|
||||
|
||||
MTX_F_JUR = {
|
||||
'Administration': 'Etabl. Public',
|
||||
'Autre': 'Autre',
|
||||
'Association': 'Association',
|
||||
'EARL': 'EARL',
|
||||
'EI': 'Individuel',
|
||||
'ESAT': 'ESAT',
|
||||
'EURL': 'EURL',
|
||||
'GAEC': 'GAEC',
|
||||
'SA': 'SA',
|
||||
'SARL': 'SARL',
|
||||
'SAS': 'SAS',
|
||||
'SCA': 'Coopérative',
|
||||
'SCEA': 'SCEA',
|
||||
'SCIC': 'SCIC',
|
||||
'SCOP': 'SCOP',
|
||||
'SNC': 'SNC',
|
||||
None: 'Inconnu',
|
||||
}
|
||||
|
||||
MTX_STATUT_STR = {'fin activite': 'ARRÊT ACTIVITÉ', '_projet': 'PROJET'}
|
||||
|
||||
# map les categories d'acteurs
|
||||
MTX_CAT_ACT = {
|
||||
'Producteur': 'cat_act_agriculteur',
|
||||
'Collecteur': 'cat_act_collecteur',
|
||||
'Transformateur': 'cat_act_transformateur',
|
||||
'Commerce de gros': 'cat_act_commerce_gros',
|
||||
'Commerce de detail': 'cat_act_commerce_detail',
|
||||
}
|
||||
|
||||
MTX_STATUTBIO = {
|
||||
None: 0,
|
||||
'fin activite': 0,
|
||||
'conventionnel': 1,
|
||||
'mixte': 2,
|
||||
'bio': 2,
|
||||
}
|
||||
|
||||
MTX_ETIQUETTE = {
|
||||
'import_abp': 'import_abp',
|
||||
'import_abp_err': 'import_abp_err',
|
||||
'import_abp_manuel': 'import_abp_manuel',
|
||||
}
|
||||
|
||||
MTX_CIVILITE = {
|
||||
'M.': 'M.',
|
||||
'Mme': 'Mme',
|
||||
'Mms.': 'Mme', # transfuge entre Mlle et Mrs
|
||||
'M. et Mme': '', # la base n'admet pas 2 personnes physiques
|
||||
}
|
||||
|
||||
MTX_ACT = {
|
||||
'autre': 'autre',
|
||||
'autre elevage': 'autre_elevage',
|
||||
'apiculture': 'apiculture',
|
||||
'fruit': 'arboriculture',
|
||||
'autre culture': 'autre_prod_veg',
|
||||
'bovin-lait': 'bovin_lait',
|
||||
'bovin-viande': 'bovin_viande',
|
||||
'caprin': 'caprin',
|
||||
'surface fourragere': 'fourrage',
|
||||
'grande culture': 'grande_culture',
|
||||
'lapin': 'lapin',
|
||||
'legume de plein champ': 'legume_pl_champ',
|
||||
'legume maraichage': 'maraichage',
|
||||
'ovin lait': 'ovin_lait',
|
||||
'ovin viande': 'ovin_viande',
|
||||
'porcin': 'porcin',
|
||||
'poule pondeuse': 'poule_pondeuse',
|
||||
'PPAM': 'ppam',
|
||||
'vigne': 'viticulture',
|
||||
'volaille de chair': 'volaille_chair',
|
||||
}
|
||||
|
||||
MTX_CULTURE = {'autre': "Autre culture (préciser)"}
|
Fichier diff supprimé car celui-ci est trop grand
Voir la Diff
|
@ -3,6 +3,7 @@ from django.core import management
|
|||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skip
|
||||
@pytest.mark.django_db
|
||||
def test_countrow_call():
|
||||
management.call_command('countrow', verbosity=0)
|
||||
|
|
|
@ -64,9 +64,13 @@ DATABASES = {
|
|||
'default': env.db(
|
||||
'DJANGO_DATABASE_URL',
|
||||
default='sqlite:///{}'.format(base_dir('sqlite.db')),
|
||||
)
|
||||
),
|
||||
'biopic': env.db(
|
||||
'DJANGO_DATABASE_URL_BIOPIC',
|
||||
default='sqlite:///{}'.format(base_dir('sqlite_biopic.db')),
|
||||
),
|
||||
}
|
||||
|
||||
DATABASE_ROUTERS = ['gaby.biopic.dbrouter.BiopicRouter']
|
||||
# URLS
|
||||
# ------------------------------------------------------------------------------
|
||||
# https://docs.djangoproject.com/en/stable/ref/settings/#root-urlconf
|
||||
|
@ -89,7 +93,12 @@ DJANGO_APPS = [
|
|||
THIRD_PARTY_APPS = ['basket', 'location_field']
|
||||
|
||||
# Project applications
|
||||
LOCAL_APPS = ['gaby.contacts', 'gaby.references', 'gaby.countrow']
|
||||
LOCAL_APPS = [
|
||||
'gaby.contacts',
|
||||
'gaby.references',
|
||||
'gaby.countrow',
|
||||
'gaby.biopic',
|
||||
]
|
||||
|
||||
# https://docs.djangoproject.com/en/stable/ref/settings/#installed-apps
|
||||
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
|
||||
|
|
|
@ -56,3 +56,6 @@ EMAIL_BACKEND = 'django.core.mail.backends.locmem.EmailBackend'
|
|||
EMAIL_HOST = 'localhost'
|
||||
# https://docs.djangoproject.com/en/stable/ref/settings/#email-port
|
||||
EMAIL_PORT = 1025
|
||||
|
||||
# don't bother create this database during tests
|
||||
del DATABASES['biopic']
|
||||
|
|
|
@ -4,3 +4,4 @@ git+https://github.com/caioariede/django-location-field.git@90718da602c09a4aeb58
|
|||
Pillow
|
||||
luhn # https://github.com/mmcloughlin/luhn (verifier les Sirets)
|
||||
django-positions ==0.6.0
|
||||
mysqlclient ==1.3.13
|
||||
|
|
|
@ -0,0 +1,22 @@
|
|||
#! /bin/bash
|
||||
|
||||
PYTHON=../venv/bin/python
|
||||
MANAGE=../manage.py
|
||||
|
||||
dump_help()
|
||||
{
|
||||
cat <<EOF
|
||||
$0 <mysql_dump>
|
||||
|
||||
ecrase la base de donnee de l'application "biopic"
|
||||
avec le dump mysql custom indiqué
|
||||
EOF
|
||||
}
|
||||
|
||||
[[ "x$1" = "x-h" ]] && { dump_help ; exit ; }
|
||||
[[ $# -lt 1 ]] && { dump_help ; exit ; }
|
||||
DUMP_FILE="$1"
|
||||
read -p "Ecraser la base de donnee 'biopic' avec le contenu de [${DUMP_FILE}] (o/n)" ANS
|
||||
[[ "$ANS" = "o" ]] || exit
|
||||
|
||||
zcat "$DUMP_FILE" | $PYTHON $MANAGE dbshell --database=biopic
|
Chargement…
Référencer dans un nouveau ticket