diff --git a/apps/livestock/management/commands/import_livestock_from_table.py b/apps/livestock/management/commands/import_livestock_from_table.py new file mode 100644 index 0000000..bacffbd --- /dev/null +++ b/apps/livestock/management/commands/import_livestock_from_table.py @@ -0,0 +1,144 @@ +from datetime import datetime + +import jdatetime +from django.core.management.base import BaseCommand +from django.db import transaction +from django.utils import timezone + +from apps.herd.models import Herd +from apps.livestock.models import ( + LiveStock, + LiveStockSpecies, + ExcelLiveStocks +) + +BATCH_SIZE = 100 + + +class Command(BaseCommand): + help = "Import livestock from ExcelLiveStocks into LiveStock using bulk_create" + + def normalize_herd_code(self, value, length=10): + if value is None: + return None + return str(value).strip().zfill(length) + + def parse_jalali_datetime(self, date_str: str): + if not date_str: + return None + + year, month, day = map(int, date_str.split('/')) + + # jalali → gregorian (date) + g_date = jdatetime.date(year, month, day).togregorian() + + # date → naive datetime + naive_dt = datetime.combine(g_date, datetime.min.time()) + + # naive → aware (VERY IMPORTANT) + return timezone.make_aware(naive_dt) + + def handle(self, *args, **options): + qs = ExcelLiveStocks.objects.all() + + if not qs.exists(): + self.stdout.write(self.style.WARNING("No records to import")) + return + + # ---------- preload lookups ---------- + herd_map = { + h.code: h + for h in Herd.objects.all() + } + + species_map = { + s.name.strip(): s + for s in LiveStockSpecies.objects.all() + } + + livestocks_to_create = [] + processed_ids = [] + + created_count = 0 + skipped = 0 + + self.stdout.write("Starting import...") + + with transaction.atomic(): + for row in qs.iterator(chunk_size=BATCH_SIZE): + herd = herd_map.get(self.normalize_herd_code(row.herd_code)) + # print(self.normalize_herd_code(row.herd_code)) + if not herd: + # print("herd") + skipped += 1 + continue + + # species cache / create + species_name = (row.species or "").strip() + if not species_name: + # print("species") + skipped += 1 + continue + + species = species_map.get(species_name) + if not species: + species = LiveStockSpecies.objects.create( + name=species_name + ) + species_map[species_name] = species + + livestocks_to_create.append( + LiveStock( + herd=herd, + species=species, + gender=self.map_gender(row.gender), + birthdate=self.parse_jalali_datetime(row.birthdate), + ) + ) + processed_ids.append(row.id) + + if len(livestocks_to_create) >= BATCH_SIZE: + print("-----------------------------CREATE------------------------------------") + print(livestocks_to_create) + LiveStock.objects.bulk_create( + livestocks_to_create, + batch_size=BATCH_SIZE + ) + created_count += len(livestocks_to_create) + livestocks_to_create.clear() + break + + # flush remaining + if livestocks_to_create: + LiveStock.objects.bulk_create( + livestocks_to_create, + batch_size=BATCH_SIZE + ) + created_count += len(livestocks_to_create) + + # mark excel rows as archived + # ExcelLiveStocks.objects.filter( + # id__in=processed_ids + # ).update(archive=True) + + self.stdout.write(self.style.SUCCESS( + f"Import finished. Created: {created_count}, Skipped: {skipped}" + )) + + @staticmethod + def map_gender(value): + if not value: + return 1 + value = value.strip().lower() + if value in ['female', 'f', 'ماده']: + return 2 + return 1 + + @staticmethod + def parse_date(value): + if not value: + return None + try: + return datetime.strptime(value, '%Y/%m/%d') + except Exception: + return None diff --git a/apps/livestock/migrations/0020_excellivestocks_sync_status.py b/apps/livestock/migrations/0020_excellivestocks_sync_status.py new file mode 100644 index 0000000..cbe182f --- /dev/null +++ b/apps/livestock/migrations/0020_excellivestocks_sync_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2026-02-10 08:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('livestock', '0019_excellivestocks'), + ] + + operations = [ + migrations.AddField( + model_name='excellivestocks', + name='sync_status', + field=models.CharField(max_length=50, null=True), + ), + ] diff --git a/apps/livestock/models.py b/apps/livestock/models.py index ea05bfe..069fc4a 100644 --- a/apps/livestock/models.py +++ b/apps/livestock/models.py @@ -110,6 +110,7 @@ class ExcelLiveStocks(BaseModel): birthdate = models.CharField(max_length=150, null=True) gender = models.CharField(max_length=150, null=True) agent_code = models.CharField(max_length=150, null=True) + sync_status = models.CharField(max_length=50, null=True) class TemporaryLiveStock(BaseModel): diff --git a/apps/tag/management/commands/sync_livestock.py b/apps/tag/management/commands/sync_livestock.py index 29f359f..65ddf09 100644 --- a/apps/tag/management/commands/sync_livestock.py +++ b/apps/tag/management/commands/sync_livestock.py @@ -4,12 +4,12 @@ from django.core.management.base import BaseCommand from django.db import transaction from apps.herd.models import Herd -from apps.livestock.models import LiveStock, LiveStockType -from apps.tag.models import Tag, TemporaryTags +from apps.livestock.models import LiveStock, LiveStockType, ExcelLiveStocks +from apps.tag.models import Tag from common.generics import parse_birthdate -BATCH_SIZE = 5000 -CHUNK_SIZE = 10000 +BATCH_SIZE = 1000 +CHUNK_SIZE = 1000 class Command(BaseCommand): @@ -22,16 +22,16 @@ class Command(BaseCommand): ) qs = ( - TemporaryTags.objects + ExcelLiveStocks.objects .filter(sync_status__isnull=True) - .only('herd_code', 'birthdate', 'gender', 'tag') + .only('herd_code', 'birthdate', 'gender', 'national_id') ) total = qs.count() processed = 0 start_time = time.time() - LOG_EVERY = 10000 + LOG_EVERY = 1000 buffer = [] for temp in qs.iterator(chunk_size=CHUNK_SIZE): @@ -64,7 +64,7 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS("DONE ✅")) def process_batch(self, temps): - herd_codes = {t.herd_code for t in temps if t.herd_code} + herd_codes = {self.normalize_herd_code(t.herd_code) for t in temps if t.herd_code} herds = { h.code: h @@ -90,7 +90,7 @@ class Command(BaseCommand): existing_tags = { t.tag_code: t for t in Tag.objects.filter( - tag_code__in=[t.tag for t in temps if t.tag] + tag_code__in=[t.national_id for t in temps if t.national_id] ) } @@ -99,28 +99,28 @@ class Command(BaseCommand): new_tags = [] for temp in temps: - herd = herds.get(temp.herd_code) + herd = herds.get(self.normalize_herd_code(temp.herd_code)) if not herd: continue birthdate = parse_birthdate(temp.birthdate) gender = 1 if temp.gender == 'M' else 2 - livestock_type = livestock_types.get(temp.type) + livestock_type = livestock_types.get(temp.species) weight_type = livestock_type.weight_type - key = (temp.herd_code, birthdate, gender) + key = (self.normalize_herd_code(temp.herd_code), birthdate, gender) livestock = livestock_map.get(key) if not livestock: - if not temp.tag: + if not temp.national_id: continue - tag = existing_tags.get(temp.tag) + tag = existing_tags.get(temp.national_id) if not tag: - tag = Tag(tag_code=temp.tag, status='A') + tag = Tag(tag_code=temp.national_id, status='A') new_tags.append(tag) - existing_tags[temp.tag] = tag + existing_tags[temp.national_id] = tag livestock = LiveStock( herd=herd, @@ -136,13 +136,13 @@ class Command(BaseCommand): temp.sync_status = 'S' continue - if livestock.tag is None and temp.tag: - tag = existing_tags.get(temp.tag) + if livestock.tag is None and temp.national_id: + tag = existing_tags.get(temp.national_id) if not tag: - tag = Tag(tag_code=temp.tag, status='A') + tag = Tag(tag_code=temp.national_id, status='A') new_tags.append(tag) - existing_tags[temp.tag] = tag + existing_tags[temp.national_id] = tag livestock.tag = tag updated_livestock.append(livestock) @@ -151,18 +151,24 @@ class Command(BaseCommand): with transaction.atomic(): Tag.objects.bulk_create(new_tags, batch_size=BATCH_SIZE) - LiveStock.objects.bulk_create( + ss = LiveStock.objects.bulk_create( new_livestock, batch_size=BATCH_SIZE, ignore_conflicts=True ) + print(ss) LiveStock.objects.bulk_update( updated_livestock, ['tag'], batch_size=BATCH_SIZE ) - TemporaryTags.objects.bulk_update( + ExcelLiveStocks.objects.bulk_update( temps, ['sync_status'], batch_size=BATCH_SIZE ) + + def normalize_herd_code(self, value, length=10): + if value is None: + return None + return str(value).strip().zfill(length)