Fix GSX parts DB upload

5 files changed, 135 insertions, 69 deletions

diff --git a/requirements.pip b/requirements.pip
index 87249c2..fc423a9 100644
--- a/requirements.pip
+++ b/requirements.pip
@@ -14,7 +14,8 @@ pytz
 phonenumbers
 python-dateutil
 python-barcode
-celery
+rq
+django-rq
 chardet
 html2text
 python-magic
diff --git a/servo/management/commands/importparts.py b/servo/management/commands/importparts.py
index 006c10d..8909773 100644
--- a/servo/management/commands/importparts.py
+++ b/servo/management/commands/importparts.py
@@ -1,103 +1,147 @@
 # -*- coding: utf-8 -*-
-
+"""
+Headers:
+    "Product Name"', 'Part Number', 'Part Description', 'Part Type',
+    'Labor Tier', 'Part Currency', 'Pricing Option', 'Price',
+    'EEE Code', 'Substitute Part', 'Component Group', 'Serialized Module'
+"""
 import re
 import os
-import logging
-
+import csv
 from decimal import Decimal, InvalidOperation, ROUND_CEILING
 
+import django_rq
+import django
+django.setup()
+
 from django.db import DatabaseError
 from django.core.management.base import BaseCommand
 from django.contrib.contenttypes.models import ContentType
 
 from servo.models import Product, TaggedItem
 
+def import_parts(parts, **options):
+    content_type = ContentType.objects.get(model="product")
 
-class Command(BaseCommand):
+    for p in parts:
+        product, created = Product.objects.get_or_create(code=p['code'])
 
-    help = "Imports complete GSX parts database"
+        product.title = p['title']
+        product.eee_code = p['eee_code']
+        product.currency = p['currency']
+        product.part_type = p['part_type']
+        product.labour_tier = p['labour_tier']
 
-    def handle(self, *args, **options):
+        product.is_serialized = p['is_serialized']
+        product.component_code = p['component_code']
 
-        update_prices = True
-        import_vintage = True
-        dbpath = "servo/uploads/products/partsdb.csv"
+        if options['update_prices']:
+            if p['stock_price']:
+                product.price_purchase_stock = p['stock_price'].to_integral_exact(
+                    rounding=ROUND_CEILING
+                )
+                product.set_stock_sales_price()
 
-        try:
-            partsdb = open(dbpath, "r")
-        except Exception:
-            pass
+            if p['exchange_price']:
+                product.price_purchase_exchange = p['exchange_price'].to_integral_exact(
+                    rounding=ROUND_CEILING
+                )
+                product.set_exchange_sales_price()
 
-        content_type = ContentType.objects.get(model="product")
+        product.save()
 
-        for l in partsdb.readlines():
+        if created:
+            try:
+                tag, _ = TaggedItem.objects.get_or_create(
+                    content_type=content_type,
+                    object_id=product.pk,
+                    tag=p['category'])
+                tag.save()
+            except DatabaseError:
+                pass
 
-            line = l.decode("iso-8859-1")
-            row = line.strip().split("\t")
+def process_parts(reader, **options):
+    parts = []
+    for row in reader:
+        category = row[0]
+        p_number = row[1]
+        p_title = row[2]
+        p_type = row[3]
+        lab_tier = row[4]
+        currency = row[5]
+        pricing_option = row[6]
+        price = row[7]
+        eee_code = row[8]
 
-            if row[5] == "" or row[5] == "Currency":
-                continue  # Skip header row and rows without currency
+        if currency == "" or row[5] == "Part Currency":
+            continue # Skip header row and rows without currency
 
-            logging.debug(row)
+        if currency != options['currency']:
+            continue # Skip unwanted currency
 
-            category = row[0]
+        if re.match(r'~VIN', category):
+            if options['import_vintage'] == False:
+                continue # Skip vintage devices if so desired
 
-            if re.match(r'~VIN', category) and not import_vintage:
-                continue  # Skip vintage devices if so desired
+        if re.match(r'675-', p_number):
+            continue # Skip DEPOT REPAIR INVOICE
 
-            p_number = row[1]
+        # skip substitute
+        component_group = row[10] or ''
 
-            if re.match(r'675-', p_number):
-                continue  # Skip DEPOT REPAIR INVOICE
+        stock_price = Decimal(0.0)
+        exchange_price = Decimal(0.0)
 
-            p_title = row[2]
-            p_type = row[3]
-            lab_tier = row[4]
+        if pricing_option.startswith("Exchange"):
+            exchange_price = Decimal(price)
 
-            try:
-                stock_price = Decimal(row[6])
-            except InvalidOperation:
-                continue  # Skip parts with no stock price
+        if pricing_option.startswith("Stocking"):
+            stock_price = Decimal(price)
 
-            exchange_price = Decimal(row[7])
+        part = {
+            'code': p_number,
+            'title': p_title,
+            'eee_code': eee_code,
+            'currency': currency,
+            'category': category,
+            'labour_tier': lab_tier,
+            'stock_price': stock_price,
+            'exchange_price': exchange_price,
+            'part_type': p_type or "OTHER",
+            'component_code': component_group,
+            'is_serialized': (row[11] == "Y")
+        }
 
-            eee_code = row[8]
+        parts.append(part)
 
-            # skip substitute
-            component_group = row[10] or None
-            is_serialized = row[11]
-            req_diag = (row[12] == "Y")
+    return parts
 
-            product, created = Product.objects.get_or_create(code=p_number)
 
-            product.title = p_title
-            product.eee_code = eee_code
-            product.labour_tier = lab_tier
-            product.part_type = p_type or "OTHER"
+class Command(BaseCommand):
 
-            product.component_code = component_group
-            product.is_serialized = (is_serialized == "Y")
+    help = "Imports complete GSX parts database"
 
-            if update_prices:
-                if stock_price:
-                    purchase_sp = Decimal(stock_price)
-                    product.price_purchase_stock = purchase_sp.to_integral_exact(rounding=ROUND_CEILING)
-                    product.set_stock_sales_price()
+    def add_arguments(self, parser):
+        parser.add_argument('dbpath', type=str)
+        parser.add_argument('currency', type=str)
 
-                if exchange_price:
-                    purchase_ep = Decimal(exchange_price)
-                    product.price_purchase_exchange = purchase_ep.to_integral_exact(rounding=ROUND_CEILING)
-                    product.set_exchange_sales_price()
+        parser.add_argument(
+            '--update_prices',
+            action='store_true',
+            help='Update part prices in inventory',
+        )
 
-            product.save()
+        parser.add_argument(
+            '--import_vintage',
+            action='store_true',
+            help='Import vintage parts',
+        )
 
-            try:
-                tag, created = TaggedItem.objects.get_or_create(
-                    content_type=content_type,
-                    object_id=product.pk,
-                    tag=category)
-                tag.save()
-            except DatabaseError:
-                pass
+    def handle(self, *args, **options):
+
+        with open(options['dbpath'], newline='') as partsdb:
+            reader = csv.reader(partsdb, delimiter=',', quotechar='"')
+            parts = process_parts(reader, **options)
+            django_rq.enqueue(import_parts, parts, **options)
 
-        os.unlink(dbpath)
+        os.unlink(options['dbpath'])
diff --git a/servo/urls/default.py b/servo/urls/default.py
index e1d86d6..00e88c2 100644
--- a/servo/urls/default.py
+++ b/servo/urls/default.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from django.urls import path
 from django.conf.urls import include, url
 from django.views.generic import RedirectView, TemplateView
 from servo.views import account, files, gsx
@@ -54,3 +55,7 @@ urlpatterns = [
     url(r'^home/', include('servo.urls.account')),
     url(r'^search/', include('servo.urls.search')),
 ]
+
+urlpatterns += [
+    path('django-rq/', include('django_rq.urls'))
+]
diff --git a/servo/views/product.py b/servo/views/product.py
index 4c7f96f..d2c6750 100644
--- a/servo/views/product.py
+++ b/servo/views/product.py
@@ -8,6 +8,7 @@ from django.db import IntegrityError
 from django.contrib import messages
 from django.core.cache import cache
 from django.http import HttpResponse
+from django.core.management import call_command
 from django.utils.translation import ugettext as _
 from django.forms.models import inlineformset_factory
 from django.contrib.contenttypes.models import ContentType
@@ -108,14 +109,20 @@ def upload_gsx_parts(request, group=None):
         form = PartsImportForm(request.POST, request.FILES)
 
         if form.is_valid():
+
             data = form.cleaned_data
-            filename = "servo/uploads/products/partsdb.csv"
+            filename = "uploads/products/partsdb.csv"
             destination = open(filename, "wb+")
 
             for chunk in data['partsdb'].chunks():
                 destination.write(chunk)
 
-            messages.success(request, _("Parts database uploaded for processing"))
+            call_command('importparts', filename, data['currency'],
+                update_prices=data['update_prices'],
+                import_vintage=data['import_vintage'],
+            )
+
+            messages.success(request, _("Parts database uploaded for processing..."))
             return redirect(list_products)
 
     data['form'] = form
diff --git a/settings.py b/settings.py
index 56090a0..2999959 100644
--- a/settings.py
+++ b/settings.py
@@ -124,7 +124,7 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'rest_framework', 'wkhtmltopdf',
     'rest_framework.authtoken',
-    'mptt', 'bootstrap3',
+    'mptt', 'bootstrap3', 'django_rq',
     'servo',
 )
 
@@ -236,3 +236,12 @@ CACHES['comptia'] = {
     'TIMEOUT': 60 * 60 * 24,
     'KEY_PREFIX': 'comptia_'
 }
+
+RQ_QUEUES = {
+    'default': {
+        'HOST': 'localhost',
+        'PORT': 6379,
+        'DB': 0,
+        'DEFAULT_TIMEOUT': 3600, # 60 minutes
+    },
+}