MNPC/RasadDam_Backend
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix --> sync livestocks with table ExcelLivestocks

Browse Source
This commit is contained in:
Mojtaba-z
2026-02-10 12:30:41 +03:30
parent 67fa1e23e7
commit 1e773ef53d
4 changed files with 191 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
apps/livestock/management/commands/import_livestock_from_table.py Normal file
View File

@@ -0,0 +1,144 @@
from datetime import datetime
import jdatetime
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils import timezone
from apps.herd.models import Herd
from apps.livestock.models import (
LiveStock,
LiveStockSpecies,
ExcelLiveStocks
)
BATCH_SIZE = 100
class Command(BaseCommand):
help = "Import livestock from ExcelLiveStocks into LiveStock using bulk_create"
def normalize_herd_code(self, value, length=10):
if value is None:
return None
return str(value).strip().zfill(length)
def parse_jalali_datetime(self, date_str: str):
if not date_str:
return None
year, month, day = map(int, date_str.split('/'))
# jalali → gregorian (date)
g_date = jdatetime.date(year, month, day).togregorian()
# date → naive datetime
naive_dt = datetime.combine(g_date, datetime.min.time())
# naive → aware (VERY IMPORTANT)
return timezone.make_aware(naive_dt)
def handle(self, *args, **options):
qs = ExcelLiveStocks.objects.all()
if not qs.exists():
self.stdout.write(self.style.WARNING("No records to import"))
return
# ---------- preload lookups ----------
herd_map = {
h.code: h
for h in Herd.objects.all()
}
species_map = {
s.name.strip(): s
for s in LiveStockSpecies.objects.all()
}
livestocks_to_create = []
processed_ids = []
created_count = 0
skipped = 0
self.stdout.write("Starting import...")
with transaction.atomic():
for row in qs.iterator(chunk_size=BATCH_SIZE):
herd = herd_map.get(self.normalize_herd_code(row.herd_code))
# print(self.normalize_herd_code(row.herd_code))
if not herd:
# print("herd")
skipped += 1
continue
# species cache / create
species_name = (row.species or "").strip()
if not species_name:
# print("species")
skipped += 1
continue
species = species_map.get(species_name)
if not species:
species = LiveStockSpecies.objects.create(
name=species_name
)
species_map[species_name] = species
livestocks_to_create.append(
LiveStock(
herd=herd,
species=species,
gender=self.map_gender(row.gender),
birthdate=self.parse_jalali_datetime(row.birthdate),
)
)
processed_ids.append(row.id)
if len(livestocks_to_create) >= BATCH_SIZE:
print("-----------------------------CREATE------------------------------------")
print(livestocks_to_create)
LiveStock.objects.bulk_create(
livestocks_to_create,
batch_size=BATCH_SIZE
)
created_count += len(livestocks_to_create)
livestocks_to_create.clear()
break
# flush remaining
if livestocks_to_create:
LiveStock.objects.bulk_create(
livestocks_to_create,
batch_size=BATCH_SIZE
)
created_count += len(livestocks_to_create)
# mark excel rows as archived
# ExcelLiveStocks.objects.filter(
# id__in=processed_ids
# ).update(archive=True)
self.stdout.write(self.style.SUCCESS(
f"Import finished. Created: {created_count}, Skipped: {skipped}"
))
@staticmethod
def map_gender(value):
if not value:
return 1
value = value.strip().lower()
if value in ['female', 'f', 'ماده']:
return 2
return 1
@staticmethod
def parse_date(value):
if not value:
return None
try:
return datetime.strptime(value, '%Y/%m/%d')
except Exception:
return None

18
apps/livestock/migrations/0020_excellivestocks_sync_status.py Normal file
View File

@@ -0,0 +1,18 @@
# Generated by Django 5.0 on 2026-02-10 08:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('livestock', '0019_excellivestocks'),
]
operations = [
migrations.AddField(
model_name='excellivestocks',
name='sync_status',
field=models.CharField(max_length=50, null=True),
),
]

1
apps/livestock/models.py
View File

@@ -110,6 +110,7 @@ class ExcelLiveStocks(BaseModel):
birthdate = models.CharField(max_length=150, null=True)
gender = models.CharField(max_length=150, null=True)
agent_code = models.CharField(max_length=150, null=True)
sync_status = models.CharField(max_length=50, null=True)
class TemporaryLiveStock(BaseModel):

50
apps/tag/management/commands/sync_livestock.py
View File

@@ -4,12 +4,12 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from apps.herd.models import Herd
from apps.livestock.models import LiveStock, LiveStockType
from apps.tag.models import Tag, TemporaryTags
from apps.livestock.models import LiveStock, LiveStockType, ExcelLiveStocks
from apps.tag.models import Tag
from common.generics import parse_birthdate
BATCH_SIZE = 5000
CHUNK_SIZE = 10000
BATCH_SIZE = 1000
CHUNK_SIZE = 1000
class Command(BaseCommand):
@@ -22,16 +22,16 @@ class Command(BaseCommand):
)
qs = (
TemporaryTags.objects
ExcelLiveStocks.objects
.filter(sync_status__isnull=True)
.only('herd_code', 'birthdate', 'gender', 'tag')
.only('herd_code', 'birthdate', 'gender', 'national_id')
)
total = qs.count()
processed = 0
start_time = time.time()
LOG_EVERY = 10000
LOG_EVERY = 1000
buffer = []
for temp in qs.iterator(chunk_size=CHUNK_SIZE):
@@ -64,7 +64,7 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS("DONE ✅"))
def process_batch(self, temps):
herd_codes = {t.herd_code for t in temps if t.herd_code}
herd_codes = {self.normalize_herd_code(t.herd_code) for t in temps if t.herd_code}
herds = {
h.code: h
@@ -90,7 +90,7 @@ class Command(BaseCommand):
existing_tags = {
t.tag_code: t
for t in Tag.objects.filter(
tag_code__in=[t.tag for t in temps if t.tag]
tag_code__in=[t.national_id for t in temps if t.national_id]
)
}
@@ -99,28 +99,28 @@ class Command(BaseCommand):
new_tags = []
for temp in temps:
herd = herds.get(temp.herd_code)
herd = herds.get(self.normalize_herd_code(temp.herd_code))
if not herd:
continue
birthdate = parse_birthdate(temp.birthdate)
gender = 1 if temp.gender == 'M' else 2
livestock_type = livestock_types.get(temp.type)
livestock_type = livestock_types.get(temp.species)
weight_type = livestock_type.weight_type
key = (temp.herd_code, birthdate, gender)
key = (self.normalize_herd_code(temp.herd_code), birthdate, gender)
livestock = livestock_map.get(key)
if not livestock:
if not temp.tag:
if not temp.national_id:
continue
tag = existing_tags.get(temp.tag)
tag = existing_tags.get(temp.national_id)
if not tag:
tag = Tag(tag_code=temp.tag, status='A')
tag = Tag(tag_code=temp.national_id, status='A')
new_tags.append(tag)
existing_tags[temp.tag] = tag
existing_tags[temp.national_id] = tag
livestock = LiveStock(
herd=herd,
@@ -136,13 +136,13 @@ class Command(BaseCommand):
temp.sync_status = 'S'
continue
if livestock.tag is None and temp.tag:
tag = existing_tags.get(temp.tag)
if livestock.tag is None and temp.national_id:
tag = existing_tags.get(temp.national_id)
if not tag:
tag = Tag(tag_code=temp.tag, status='A')
tag = Tag(tag_code=temp.national_id, status='A')
new_tags.append(tag)
existing_tags[temp.tag] = tag
existing_tags[temp.national_id] = tag
livestock.tag = tag
updated_livestock.append(livestock)
@@ -151,18 +151,24 @@ class Command(BaseCommand):
with transaction.atomic():
Tag.objects.bulk_create(new_tags, batch_size=BATCH_SIZE)
LiveStock.objects.bulk_create(
ss = LiveStock.objects.bulk_create(
new_livestock,
batch_size=BATCH_SIZE,
ignore_conflicts=True
)
print(ss)
LiveStock.objects.bulk_update(
updated_livestock,
['tag'],
batch_size=BATCH_SIZE
)
TemporaryTags.objects.bulk_update(
ExcelLiveStocks.objects.bulk_update(
temps,
['sync_status'],
batch_size=BATCH_SIZE
)
def normalize_herd_code(self, value, length=10):
if value is None:
return None
return str(value).strip().zfill(length)