diff --git a/apps/herd/management/commands/sync_herd_rancher.py b/apps/herd/management/commands/sync_herd_rancher.py new file mode 100644 index 0000000..02ef4d8 --- /dev/null +++ b/apps/herd/management/commands/sync_herd_rancher.py @@ -0,0 +1,70 @@ +from django.core.management.base import BaseCommand +from django.db import transaction + +from apps.herd.models import ( + HerdRancherTemporary, +) +from apps.herd.services.herd_rancher_sync import HerdRancherSyncService + + +class Command(BaseCommand): + help = "Bulk Sync Rancher & Herd from HerdRancherTemporary" + + def add_arguments(self, parser): + parser.add_argument( + '--limit', + type=int, + help='limit number of records' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='run without saving data' + ) + parser.add_argument( + '--batch-size', + type=int, + default=1000, + help='batch size' + ) + + def handle(self, *args, **options): + limit = options['limit'] + dry_run = options['dry_run'] + batch_size = options['batch_size'] + + qs = HerdRancherTemporary.objects.all() + + if limit: + qs = qs[:limit] + + total = qs.count() + + self.stdout.write( + self.style.NOTICE(f"Start bulk syncing {total} records") + ) + + if dry_run: + self.stdout.write( + self.style.WARNING("Running in DRY-RUN mode (no DB writes)") + ) + return + + try: + with transaction.atomic(): + HerdRancherSyncService.bulk_sync( + queryset=qs, + batch_size=batch_size + ) + + qs.update(sync_status='S') + + except Exception as e: + qs.update(sync_status='F') + raise e + + self.stdout.write( + self.style.SUCCESS( + f"Done. synced={total}" + ) + ) diff --git a/apps/herd/migrations/0025_herdranchertemporary_sync_status.py b/apps/herd/migrations/0025_herdranchertemporary_sync_status.py new file mode 100644 index 0000000..3795d6d --- /dev/null +++ b/apps/herd/migrations/0025_herdranchertemporary_sync_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2025-12-28 05:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('herd', '0024_alter_herdranchertemporary_agent_code_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='herdranchertemporary', + name='sync_status', + field=models.CharField(max_length=50, null=True), + ), + ] diff --git a/apps/herd/migrations/0026_herdranchertemporary_city.py b/apps/herd/migrations/0026_herdranchertemporary_city.py new file mode 100644 index 0000000..a7f8b65 --- /dev/null +++ b/apps/herd/migrations/0026_herdranchertemporary_city.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2025-12-28 06:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('herd', '0025_herdranchertemporary_sync_status'), + ] + + operations = [ + migrations.AddField( + model_name='herdranchertemporary', + name='city', + field=models.CharField(max_length=150, null=True), + ), + ] diff --git a/apps/herd/migrations/0027_alter_herd_name_alter_herd_photo.py b/apps/herd/migrations/0027_alter_herd_name_alter_herd_photo.py new file mode 100644 index 0000000..e49660c --- /dev/null +++ b/apps/herd/migrations/0027_alter_herd_name_alter_herd_photo.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0 on 2025-12-28 06:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('herd', '0026_herdranchertemporary_city'), + ] + + operations = [ + migrations.AlterField( + model_name='herd', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='herd', + name='photo', + field=models.CharField(max_length=200, null=True), + ), + ] diff --git a/apps/herd/models.py b/apps/herd/models.py index 451970f..7fda366 100644 --- a/apps/herd/models.py +++ b/apps/herd/models.py @@ -24,8 +24,8 @@ class Herd(BaseModel): related_name='herd', null=True ) - name = models.CharField(max_length=50) - photo = models.CharField(max_length=50, null=True) + name = models.CharField(max_length=200) + photo = models.CharField(max_length=200, null=True) code = models.CharField(max_length=20) heavy_livestock_number = models.BigIntegerField(default=0) light_livestock_number = models.BigIntegerField(default=0) @@ -182,6 +182,8 @@ class HerdRancherTemporary(BaseModel): mobile = models.CharField(max_length=150, null=True) agent_code = models.CharField(max_length=150, null=True) registerer_user = models.CharField(max_length=150, null=True) + sync_status = models.CharField(max_length=50, null=True) + city = models.CharField(max_length=150, null=True) def save(self, *args, **kwargs): return super(HerdRancherTemporary, self).save(*args, **kwargs) diff --git a/apps/herd/services/herd_rancher_sync.py b/apps/herd/services/herd_rancher_sync.py new file mode 100644 index 0000000..17650e8 --- /dev/null +++ b/apps/herd/services/herd_rancher_sync.py @@ -0,0 +1,132 @@ +from django.db import transaction + +from apps.authentication.models import City +from apps.herd.models import Rancher, Herd + + +class HerdRancherSyncService: + + @classmethod + def bulk_sync(cls, queryset, batch_size=1000): + """ + optimized bulk sync for large datasets + """ + + # ------------------------- + # Cache Cities + # ------------------------- + city_map = { + name.strip(): id + for id, name in City.objects.all().values_list('id', 'name') + } + + # ------------------------- + # Cache existing ranchers + # ------------------------- + rancher_map = { + r.national_code: r + for r in Rancher.objects.filter( + national_code__in=queryset.values_list( + 'rancher_national_code', flat=True + ) + ).only('id', 'national_code') + } + + new_ranchers = [] + new_herds = [] + + existing_herds = set( + Herd.objects.filter( + rancher__national_code__in=queryset.values_list( + 'rancher_national_code', flat=True + ) + ).values_list( + 'rancher__national_code', 'code' + ) + ) + + seen_in_batch = set() + + for temp in queryset.iterator(chunk_size=batch_size): + + # ------------------------- + # Rancher + # ------------------------- + rancher = rancher_map.get(temp.rancher_national_code) + + if not rancher: + rancher = Rancher( + first_name=temp.rancher_name, + mobile=temp.mobile, + national_code=temp.rancher_national_code, + rancher_type='N', + city_id=city_map.get(temp.city.strip()), + province_id=30 + ) + new_ranchers.append(rancher) + rancher_map[temp.rancher_national_code] = rancher + + # ------------------------- + # Herd + # ------------------------- + herd_key = (temp.rancher_national_code, temp.herd_code) + + if herd_key in existing_herds: + continue + + if herd_key in seen_in_batch: + continue + + seen_in_batch.add(herd_key) + + new_herds.append({ + "rancher_code": temp.rancher_national_code, + "herd": Herd( + name=temp.herd_name, + code=temp.herd_code, + epidemiologic=temp.epidemiologic, + latitude=temp.latitude, + longitude=temp.longitude, + postal=temp.postal_code, + unit_unique_id=temp.unit_unique_id, + city_id=city_map.get(temp.city.strip()), + province_id=30 + ) + } + ) + + # ------------------------- + # Bulk DB Operations + # ------------------------- + with transaction.atomic(): + Rancher.objects.bulk_create( + new_ranchers, + ignore_conflicts=True, + batch_size=batch_size + ) + + # refresh ranchers with ids + rancher_map = { + r.national_code: r + for r in Rancher.objects.filter( + national_code__in=rancher_map.keys() + ) + } + + final_herds = [] + + for item in new_herds: + rancher = rancher_map.get(item["rancher_code"]) + + if not rancher: + continue # یا raise error + + herd = item["herd"] + herd.rancher = rancher + final_herds.append(herd) + + Herd.objects.bulk_create( + final_herds, + ignore_conflicts=True, + batch_size=batch_size + ) diff --git a/apps/tag/management/commands/sync_livestock.py b/apps/tag/management/commands/sync_livestock.py new file mode 100644 index 0000000..3b8747c --- /dev/null +++ b/apps/tag/management/commands/sync_livestock.py @@ -0,0 +1,173 @@ +import time + +from django.core.management.base import BaseCommand +from django.db import transaction + +from apps.herd.models import Herd +from apps.livestock.models import LiveStock, LiveStockType +from apps.tag.models import Tag, TemporaryTags +from common.generics import parse_birthdate + +BATCH_SIZE = 5000 +CHUNK_SIZE = 10000 + + +class Command(BaseCommand): + help = "Create Livestock if missing and assign Tag if missing" + + def handle(self, *args, **options): + + self.stdout.write( + self.style.NOTICE(f"Start bulk syncing .... records") + ) + + qs = ( + TemporaryTags.objects + .filter(sync_status__isnull=True) + .only('herd_code', 'birthdate', 'gender', 'tag') + ) + + total = qs.count() + processed = 0 + start_time = time.time() + + LOG_EVERY = 10_000 # هر چند رکورد لاگ بده + + buffer = [] + for temp in qs.iterator(chunk_size=CHUNK_SIZE): + buffer.append(temp) + + if len(buffer) >= BATCH_SIZE: + self.process_batch(buffer) + processed += len(buffer) + buffer.clear() + + if processed % LOG_EVERY == 0: + elapsed = time.time() - start_time + speed = processed / elapsed if elapsed else 0 + percent = (processed / total) * 100 + remaining = total - processed + eta = remaining / speed if speed else 0 + + self.stdout.write( + self.style.NOTICE( + f"[SYNC] {processed:,}/{total:,} " + f"({percent:.2f}%) | " + f"{speed:.0f} rec/s | " + f"ETA: {eta / 60:.1f} min" + ) + ) + + if buffer: + self.process_batch(buffer) + + self.stdout.write(self.style.SUCCESS("DONE ✅")) + + # ---------------------------------------------------- + + def process_batch(self, temps): + herd_codes = {t.herd_code for t in temps if t.herd_code} + + herds = { + h.code: h + for h in Herd.objects.filter(code__in=herd_codes) + } + + livestocks = LiveStock.objects.filter( + herd__code__in=herd_codes + ).select_related('herd').only( + 'id', 'herd_id', 'birthdate', 'gender', 'tag' + ) + + livestock_map = { + (ls.herd.code, ls.birthdate, ls.gender): ls + for ls in livestocks + } + + livestock_types = { + stock_type.name: stock_type + for stock_type in LiveStockType.objects.all() + } + + existing_tags = { + t.tag_code: t + for t in Tag.objects.filter( + tag_code__in=[t.tag for t in temps if t.tag] + ) + } + + new_livestock = [] + updated_livestock = [] + new_tags = [] + + for temp in temps: + herd = herds.get(temp.herd_code) + if not herd: + continue # گله باید وجود داشته باشد + + birthdate = parse_birthdate(temp.birthdate) + gender = 1 if temp.gender == 'M' else 2 + livestock_type = livestock_types.get(temp.type) + weight_type = livestock_type.weight_type + + key = (temp.herd_code, birthdate, gender) + livestock = livestock_map.get(key) + + # ---------- دام وجود ندارد ---------- + if not livestock: + if not temp.tag: + continue + + tag = existing_tags.get(temp.tag) + + if not tag: + tag = Tag(tag_code=temp.tag, status='A') + new_tags.append(tag) + existing_tags[temp.tag] = tag + + livestock = LiveStock( + herd=herd, + birthdate=birthdate, + gender=gender, + tag=tag, + weight_type=weight_type, + type=livestock_type + ) + new_livestock.append(livestock) + livestock_map[key] = livestock + + temp.sync_status = 'S' + continue + + # ---------- دام وجود دارد ولی پلاک ندارد ---------- + if livestock.tag is None and temp.tag: + tag = existing_tags.get(temp.tag) + + if not tag: + tag = Tag(tag_code=temp.tag, status='A') + new_tags.append(tag) + existing_tags[temp.tag] = tag + + livestock.tag = tag + updated_livestock.append(livestock) + + temp.sync_status = 'S' + + # ---------- BULK ---------- + with transaction.atomic(): + Tag.objects.bulk_create(new_tags, batch_size=BATCH_SIZE) + LiveStock.objects.bulk_create( + new_livestock, + batch_size=BATCH_SIZE, + ignore_conflicts=True + ) + LiveStock.objects.bulk_update( + updated_livestock, + ['tag'], + batch_size=BATCH_SIZE + ) + TemporaryTags.objects.bulk_update( + temps, + ['sync_status'], + batch_size=BATCH_SIZE + ) diff --git a/apps/tag/migrations/0026_temporarytags_sync_status.py b/apps/tag/migrations/0026_temporarytags_sync_status.py new file mode 100644 index 0000000..53b8b11 --- /dev/null +++ b/apps/tag/migrations/0026_temporarytags_sync_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2025-12-28 10:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0025_temporarytags_agriculture_unique_id_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='temporarytags', + name='sync_status', + field=models.CharField(max_length=150, null=True), + ), + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index c97e65c..7a0cbfd 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -135,6 +135,7 @@ class TemporaryTags(BaseModel): agriculture_unique_id = models.CharField(max_length=150, null=True) keeper_agent = models.CharField(max_length=150, null=True) registerer_user = models.CharField(max_length=150, null=True) + sync_status = models.CharField(max_length=150, null=True) def save(self, *args, **kwargs): super(TemporaryTags, self).save(*args, **kwargs) diff --git a/apps/tag/services/temporary_tags_sync.py b/apps/tag/services/temporary_tags_sync.py new file mode 100644 index 0000000..e594909 --- /dev/null +++ b/apps/tag/services/temporary_tags_sync.py @@ -0,0 +1,105 @@ +from apps.tags.models import Tag +from django.db import transaction + +from apps.herd.models import Herd +from apps.livestock.models import LiveStock, LiveStockType +from apps.tag.models import TemporaryTags + +BATCH_SIZE = 1000 + + +class TemporaryTagsSyncService: + + @classmethod + @transaction.atomic + def sync_tags_to_livestock(cls): + # Preload herds + herds = {h.code: h for h in Herd.objects.all()} + + # Preload existing LiveStocks for speed + live_stocks_map = {} + for ls in LiveStock.objects.select_related('herd', 'type').all(): + key = (ls.herd.code, ls.birthdate.date() if ls.birthdate else None, ls.gender) + live_stocks_map.setdefault(key, []).append(ls) + + # Preload LiveStockType + gender_type_map = {t.name.lower(): t for t in LiveStockType.objects.all()} + + # Preload existing tags + existing_tags = {t.tag_code: t for t in Tag.objects.all()} + + new_herds = [] + new_livestock = [] + new_tags = [] + + for temp in TemporaryTags.objects.iterator(): + herd = herds.get(temp.herd_code) + if not herd: + herd = Herd(code=temp.herd_code, name=temp.herd_code, province_id=30) + new_herds.append(herd) + herds[temp.herd_code] = herd + + # Gender mapping + gender_lower = temp.gender.lower() if temp.gender else 'male' + ls_type = gender_type_map.get(gender_lower) + + birthdate = temp.birthdate # convert to datetime if needed + + key = (herd.code, birthdate, ls_type.id if ls_type else None) + live_stock_list = live_stocks_map.get(key, []) + + if live_stock_list: + # دام موجود است، فقط Tag ایجاد می‌کنیم + ls = live_stock_list[0] # می‌توانیم چندتا هم داشته باشیم + else: + # دام جدید ایجاد می‌کنیم + ls = LiveStock( + herd=herd, + type=ls_type, + gender=1 if gender_lower == 'male' else 2, + birthdate=birthdate + ) + new_livestock.append(ls) + live_stocks_map[key] = [ls] + + # ---------------------- + # Tag برای هر دام + # ---------------------- + if temp.serial: # یا temp.tag_code + tag_code = f"{temp.country_code}{temp.static_code}{temp.ownership_code}{temp.species_code}{temp.serial}" + if tag_code not in existing_tags: + tag = Tag( + serial=temp.serial, + country_code=getattr(temp, 'country_code', 364), + static_code=getattr(temp, 'static_code', 0), + ownership_code=getattr(temp, 'ownership_code', 0), + species_code=getattr(temp, 'species_code', 0), + tag_code=tag_code, + status='F' + ) + new_tags.append(tag) + existing_tags[tag_code] = tag + else: + tag = existing_tags[tag_code] + + # Assign Tag to LiveStock + ls.tag = tag + + # Batch insert + if len(new_livestock) >= BATCH_SIZE: + cls._bulk_insert(new_herds, new_livestock, new_tags) + new_herds.clear() + new_livestock.clear() + new_tags.clear() + + # insert remaining + cls._bulk_insert(new_herds, new_livestock, new_tags) + + @staticmethod + def _bulk_insert(herds, livestock, tags): + if herds: + Herd.objects.bulk_create(herds, ignore_conflicts=True, batch_size=BATCH_SIZE) + if tags: + Tag.objects.bulk_create(tags, ignore_conflicts=True, batch_size=BATCH_SIZE) + if livestock: + LiveStock.objects.bulk_create(livestock, ignore_conflicts=True, batch_size=BATCH_SIZE) diff --git a/common/generics.py b/common/generics.py index 18e8807..ae45661 100644 --- a/common/generics.py +++ b/common/generics.py @@ -1,6 +1,11 @@ -from django.core.files.base import ContentFile -from django.db.models.functions import TruncDate import base64 +from datetime import datetime +from functools import lru_cache + +import jdatetime +from django.core.files.base import ContentFile +from django.utils import timezone +from jdatetime import datetime as jalali_datetime def base64_to_image_file(base64_string, filename="image.jpg"): @@ -14,3 +19,40 @@ def base64_to_image_file(base64_string, filename="image.jpg"): def get_client_ip(request): # noqa forwarded = request.META.get('HTTP_X_FORWARDED_FOR') return forwarded.split(',')[0] if forwarded else request.META.get('REMOTE_ADDR') + + +def shamsi_to_gregorian(shamsi_str: str) -> datetime: + """ + Convert Shamsi string YYYY/MM/DD to Gregorian datetime + """ + try: + year, month, day = map(int, shamsi_str.split('/')) + jalali_date = jdatetime.date(year, month, day) + gregorian_date = jalali_date.togregorian() + return datetime(gregorian_date.year, gregorian_date.month, gregorian_date.day) + except Exception as e: + return None + + +@lru_cache(maxsize=50000) +def shamsi_to_gregorian_cached(shamsi: str): + try: + y, m, d = map(int, shamsi.split('/')) + return jdatetime.date(y, m, d).togregorian() + except: + return None + + +def parse_birthdate(jalali_str): + if not jalali_str: + return None + + gregorian_dt = jalali_datetime.strptime( + jalali_str, + "%Y/%m/%d" + ).togregorian() + + return timezone.make_aware( + gregorian_dt, + timezone.get_current_timezone() + )