From ce00cc2ffbb78604f7c2e6ba7ed528df51dc10ea Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Tue, 6 Feb 2024 15:30:44 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20[FEAT]=20Add=20fetching=20related?= =?UTF-8?q?=20categories=20and=20parent=20sites=20in=20GeotrekSiteParser?= =?UTF-8?q?=20(refs=20#3569)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- geotrek/outdoor/parsers.py | 93 +++++++++++++++++-- .../geotrek_parser_v2/outdoor_rating.json | 6 +- geotrek/outdoor/tests/test_parsers.py | 31 ++++--- 3 files changed, 106 insertions(+), 24 deletions(-) diff --git a/geotrek/outdoor/parsers.py b/geotrek/outdoor/parsers.py index 2e3ae5bbd2..c979766ad2 100644 --- a/geotrek/outdoor/parsers.py +++ b/geotrek/outdoor/parsers.py @@ -1,5 +1,6 @@ +from django.conf import settings from geotrek.common.parsers import (ApidaeBaseParser, AttachmentParserMixin, GeotrekParser, GlobalImportError, Parser) -from geotrek.outdoor.models import Site +from geotrek.outdoor.models import Practice, Rating, RatingScale, Sector, Site class GeotrekSiteParser(GeotrekParser): @@ -9,11 +10,10 @@ class GeotrekSiteParser(GeotrekParser): model = Site replace_fields = { "eid": "uuid", - "geom": "geometry" + "geom": "geometry", + "scale": "ratingscale" } url_categories = { - "practice": "outdoor_practice", - "ratings": "outdoor_rating", "themes": "theme", "type": "outdoor_sitetype", 'labels': 'label', @@ -23,7 +23,10 @@ class GeotrekSiteParser(GeotrekParser): } categories_keys_api_v2 = { "practice": "name", - "ratings": "name", + "sector": "name", + "rating": "name", + "scale": "name", + "ratingscale": "name", "themes": "label", "type": "name", 'labels': 'name', @@ -33,7 +36,6 @@ class GeotrekSiteParser(GeotrekParser): } natural_keys = { "practice": "name", - "ratings": "name", "themes": "label", "type": "name", 'labels': 'name', @@ -41,13 +43,84 @@ class GeotrekSiteParser(GeotrekParser): 'managers': 'organism', 'structure': 'name', } + parents = {} + + def get_id_from_mapping(self, mapping, value): + for dest_id, source_id in mapping.items(): + if source_id == value: + return dest_id + return None + + def init_outdoor_category(self, category, model, join_field=None, extra_fields={}): + response = self.request_or_retry(f"{self.url}/api/v2/outdoor_{category}") + results = response.json().get('results', []) + if category not in self.field_options.keys(): + self.field_options[category] = {} + if "mapping" not in self.field_options[category].keys(): + self.field_options[category]["mapping"] = {} + for result in results: + label = result["name"] + if isinstance(label, dict): + if label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE]: + replaced_label = self.replace_mapping(label[settings.MODELTRANSLATION_DEFAULT_LANGUAGE], f'outdoor_{category}') + else: + if label: + replaced_label = self.replace_mapping(label, f'outdoor_{category}') + fields = {} + for field in extra_fields: + if isinstance(result[field], dict): + if result[field][settings.MODELTRANSLATION_DEFAULT_LANGUAGE]: + fields[field] = result[field][settings.MODELTRANSLATION_DEFAULT_LANGUAGE] + else: + fields[field] = result[field] + if join_field: + mapping_key = self.replace_fields.get(join_field, join_field) + mapped_value = self.get_id_from_mapping(self.field_options[mapping_key]["mapping"], result[join_field]) + if not mapped_value: + continue # Ignore some results if related category was not retrieved + fields[f"{join_field}_id"] = mapped_value + category_obj, _ = model.objects.update_or_create(**{'name': replaced_label}, defaults=fields) + self.field_options[category]["mapping"][category_obj.pk] = result['id'] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.init_outdoor_category('sector', Sector) + self.init_outdoor_category('practice', Practice, join_field='sector') + self.init_outdoor_category('ratingscale', RatingScale, join_field='practice') + self.init_outdoor_category('rating', Rating, join_field='scale', extra_fields=['description', 'order', 'color']) self.next_url = f"{self.url}/api/v2/outdoor_site" - print("AFTER INIT 9999999999999999999999999999999999999999999999999999999") + + def filter_practice(self, src, val): + if val: + practice_id = self.get_id_from_mapping(self.field_options["practice"]["mapping"], val) + if practice_id: + return Practice.objects.get(pk=practice_id) + return None + + def filter_ratings(self, src, val): + ratings = [] + for subval in val: + rating_id = self.get_id_from_mapping(self.field_options["rating"]["mapping"], subval) + if rating_id: + ratings.append(Rating.objects.get(pk=rating_id).pk) + return ratings + + def parse_row(self, row): + super().parse_row(row) + self.parents[row['uuid']] = row['parent_uuid'] def end(self): - """Add children after all treks imported are created in database.""" - #super().end() - print("MAKE LINK BETWEEN SITES") + """Add children after all Sites imported are created in database.""" + for child, parent in self.parents.items(): + try: + parent_site = Site.objects.get(eid=parent) + except Site.DoesNotExist: + self.add_warning(f"Trying to retrieve missing parent (UUID: {parent}) for child Site (UUID: {child})") + continue + try: + child_site = Site.objects.get(eid=child) + except Site.DoesNotExist: + self.add_warning(f"Trying to retrieve missing child (UUID: {child}) for parent Site (UUID: {parent})") + continue + child_site.parent = parent_site + child_site.save() diff --git a/geotrek/outdoor/tests/data/geotrek_parser_v2/outdoor_rating.json b/geotrek/outdoor/tests/data/geotrek_parser_v2/outdoor_rating.json index 031ef22db1..310f13e89d 100644 --- a/geotrek/outdoor/tests/data/geotrek_parser_v2/outdoor_rating.json +++ b/geotrek/outdoor/tests/data/geotrek_parser_v2/outdoor_rating.json @@ -10,12 +10,12 @@ "fr": "3+" }, "description": { - "en": null, - "fr": null + "en": "A description", + "fr": "Une description" }, "scale": 6, "order": 302, - "color": "#D9D9D9" + "color": "#D9D9D8" }, { "id": 36, diff --git a/geotrek/outdoor/tests/test_parsers.py b/geotrek/outdoor/tests/test_parsers.py index d56cf2a942..b3278e4866 100644 --- a/geotrek/outdoor/tests/test_parsers.py +++ b/geotrek/outdoor/tests/test_parsers.py @@ -19,7 +19,7 @@ from geotrek.common.models import Organism, Theme, FileType, Attachment, Label from geotrek.common.tests.mixins import GeotrekParserTestMixin from geotrek.core.tests.factories import PathFactory -from geotrek.outdoor.models import Site +from geotrek.outdoor.models import Practice, Rating, RatingScale, Sector, Site from geotrek.outdoor.parsers import GeotrekSiteParser from geotrek.trekking.tests.factories import RouteFactory from geotrek.trekking.models import POI, POIType, Service, Trek, DifficultyLevel, Route @@ -29,15 +29,12 @@ ) - class TestGeotrekSiteParser(GeotrekSiteParser): url = "https://test.fr" provider = 'geotrek1' field_options = { 'type': {'create': True, }, - 'ratings': {'create': True, }, 'themes': {'create': True}, - 'practice': {'create': True}, 'geom': {'required': True}, 'labels': {'create': True}, 'source': {'create': True}, @@ -72,7 +69,6 @@ class TestGeotrek2SiteParser(GeotrekSiteParser): # 'geom': {'required': True}, # } - @override_settings(MODELTRANSLATION_DEFAULT_LANGUAGE="fr") @skipIf(settings.TREKKING_TOPOLOGY_ENABLED, 'Test without dynamic segmentation only') class SiteGeotrekParserTests(GeotrekParserTestMixin, TestCase): @@ -85,14 +81,16 @@ def setUpTestData(cls): @mock.patch('requests.head') def test_create(self, mocked_head, mocked_get): self.mock_time = 0 - self.mock_json_order = [('outdoor', 'outdoor_practice.json'), - ('outdoor', 'outdoor_rating.json'), - ('outdoor', 'theme.json'), + self.mock_json_order = [('outdoor', 'theme.json'), ('outdoor', 'outdoor_sitetype.json'), ('outdoor', 'label.json'), ('outdoor', 'source.json'), ('outdoor', 'organism.json'), ('outdoor', 'structure.json'), + ('outdoor', 'outdoor_sector.json'), + ('outdoor', 'outdoor_practice.json'), + ('outdoor', 'outdoor_ratingscale.json'), + ('outdoor', 'outdoor_rating.json'), ('outdoor', 'outdoor_site_ids.json'), ('outdoor', 'outdoor_site.json')] @@ -104,6 +102,10 @@ def test_create(self, mocked_head, mocked_get): call_command('import', 'geotrek.outdoor.tests.test_parsers.TestGeotrekSiteParser', verbosity=0) self.assertEqual(Site.objects.count(), 6) + self.assertEqual(Sector.objects.count(), 2) + self.assertEqual(RatingScale.objects.count(), 1) + self.assertEqual(Rating.objects.count(), 3) + self.assertEqual(Practice.objects.count(), 1) site = Site.objects.get(name_fr="Racine", name_en="Root") # TODO : all the ones that are commented do not work self.assertEqual(site.published, True) @@ -111,10 +113,15 @@ def test_create(self, mocked_head, mocked_get): self.assertEqual(site.published_en, True) self.assertEqual(site.published_it, False) self.assertEqual(site.published_es, False) + self.assertEqual(str(site.practice.sector), 'Vertical') self.assertEqual(str(site.practice), 'Escalade') self.assertEqual(str(site.labels.first()), 'Label fr') - #self.assertEqual(str(site.ratings.all()), 'Très facile') - #self.assertEqual(str(site.practice.sector), 'Vertical') + self.assertEqual(site.ratings.count(), 3) + self.assertEqual(str(site.ratings.first()), 'Cotation : 3+') + self.assertEqual(site.ratings.first().description, 'Une description') + self.assertEqual(site.ratings.first().order, 302) + self.assertEqual(site.ratings.first().color, '#D9D9D8') + self.assertEqual(str(site.ratings.first().scale), 'Cotation (Escalade)') self.assertEqual(str(site.type), 'Ecole') self.assertAlmostEqual(site.geom[0][0][0][0], 970023.8976707931, places=5) self.assertAlmostEqual(site.geom[0][0][0][1], 6308806.903248067, places=5) @@ -135,14 +142,16 @@ def test_create(self, mocked_head, mocked_get): self.assertEqual(site.orientation, ['NE', 'S']) self.assertEqual(site.ambiance, "Test ambiance fr") self.assertEqual(site.ambiance_en, "Test ambiance en") - #self.assertEqual(site.parent) # TODO use other to test this self.assertEqual(site.wind, ['N', 'E']) + self.assertEqual(str(site.structure), 'Test structure') # self.assertEqual(site.information_desks.count(), 1) # self.assertEqual(site.weblink.count(), 1) # self.assertEqual(site.excluded_pois.count(), 1) self.assertEqual(site.eid, "57a8fb52-213d-4dce-8224-bc997f892aae") # self.assertEqual(Attachment.objects.filter(object_id=site.pk).count(), 3) # self.assertEqual(Attachment.objects.get(object_id=site.pk, license__isnull=False).license.label, "License") + child_site = Site.objects.get(name_fr="Noeud 1", name_en="Node") + self.assertEqual(child_site.parent, site) @mock.patch('requests.get') @mock.patch('requests.head')