[error-report-web][PATCH 0/1] purge.py: Create script that will purge database

[error-report-web][PATCH 0/1] purge.py: Create script that will purge database

[Amanda Brindle]

The following changes since commit 9c9e34d0fdcc543945951e374e75b8678e71c35d:

  latest-errors: Use the key value instead of display value for filter (2016-06-23 12:24:17 +0100)

are available in the git repository at:

  git://push.yoctoproject.org/error-report-web abrindle/purge_script

Amanda Brindle (1):
  purge.py: Create script that will purge database

 Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++
 Post/models.py                               | 10 ++++++++++
 Post/purge.py                                | 29 ++++++++++++++++++++++++++++
 Post/views.py                                | 16 +++++++++++++--
 4 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 Post/migrations/0006_buildfailure_referer.py
 create mode 100644 Post/purge.py

-- 
2.7.4

[error-report-web][PATCH 1/1] purge.py: Create script that will purge database

[Amanda Brindle]

The script will be run on a regular basis to get rid of old reports that
we don't need. This will improve the performance of the application
since the database has grown to a huge size.

The script will remove reports older than thirty days if they have not
been referred to by a host other than the Error Reporting Tool.

The function details() in views.py will keep trick of the referer when
a build failure report is accessed. If there is no referer, we will note
that in the database in order to determine how often there is no
referer. It's possible that crawlers will access reports with no
referer; if that's the case, a lack of a referer should not determine
whether we save an older report or not.

Fixes [YOCTO #12332]

Signed-off-by: Amanda Brindle <amanda.r.brindle@intel.com>
---
 Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++
 Post/models.py                               | 10 ++++++++++
 Post/purge.py                                | 29 ++++++++++++++++++++++++++++
 Post/views.py                                | 16 +++++++++++++--
 4 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 Post/migrations/0006_buildfailure_referer.py
 create mode 100644 Post/purge.py

diff --git a/Post/migrations/0006_buildfailure_referer.py b/Post/migrations/0006_buildfailure_referer.py
new file mode 100644
index 0000000..5fad048
--- /dev/null
+++ b/Post/migrations/0006_buildfailure_referer.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('Post', '0005_build_error_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='buildfailure',
+            name='REFERER',
+            field=models.CharField(default=b'NOT_VISITED', max_length=14, choices=[(b'NO_REFERER', b'no_referer'), (b'OTHER', b'other'), (b'NOT_VISITED', b'not_visited')]),
+        ),
+    ]
diff --git a/Post/models.py b/Post/models.py
index ddf2fc7..fcf53ca 100644
--- a/Post/models.py
+++ b/Post/models.py
@@ -59,6 +59,16 @@ class BuildFailure(models.Model):
     ERROR_DETAILS = models.TextField(max_length=int(settings.MAX_UPLOAD_SIZE))
     BUILD = models.ForeignKey(Build)
     LEV_DISTANCE = models.IntegerField(blank=True, null=True)
+    REFERER_CHOICES = (
+            ('NO_REFERER', 'no_referer'),
+            ('OTHER', 'other'),
+            ('NOT_VISITED', 'not_visited')
+    )
+    REFERER = models.CharField(
+            max_length = 14,
+            choices = REFERER_CHOICES,
+            default = 'NOT_VISITED'
+    )
 
     def get_similar_fails(self):
         if self.LEV_DISTANCE is None:
diff --git a/Post/purge.py b/Post/purge.py
new file mode 100644
index 0000000..829fada
--- /dev/null
+++ b/Post/purge.py
@@ -0,0 +1,29 @@
+from datetime import datetime
+from django.utils import timezone
+import os
+import sys
+
+def setup_django():
+    import django
+    # Get access to our Django model
+    newpath = os.path.abspath(os.path.dirname(__file__)) + '/..'
+    sys.path.append(newpath)
+    os.environ['DJANGO_SETTINGS_MODULE'] = 'project.settings'
+    django.setup()
+
+def main():
+    setup_django()
+    from Post.models import BuildFailure
+    items = BuildFailure.objects.all()
+    now = timezone.now()
+    for item in items:
+        if item.REFERER == 'OTHER' or item.REFERER == 'NO_REFERER':
+            continue
+        difference = now - item.BUILD.DATE
+        if difference.days > 30:
+            item.delete()
+
+if __name__ == "__main__":
+    main()
+
+
diff --git a/Post/views.py b/Post/views.py
index 7f2cffb..2ea94a7 100644
--- a/Post/views.py
+++ b/Post/views.py
@@ -22,6 +22,7 @@ from django.http import JsonResponse
 from django.db.models import Q
 import json
 import urllib
+from urlparse import urlparse
 
 class results_mode(object):
     LATEST = 0
@@ -256,9 +257,20 @@ def search(request, mode=results_mode.LATEST, **kwargs):
 
 def details(request, fail_id):
     try:
-      build_failure = BuildFailure.objects.get(id=fail_id)
+        build_failure = BuildFailure.objects.get(id=fail_id)
     except ObjectDoesNotExist:
-      build_failure = None
+        build_failure = None
+    try:
+        referer = urlparse(request.META['HTTP_REFERER'])
+        referer_hostname = referer.hostname
+        if referer.port:
+            referer_hostname += ":" + str(referer.port)
+        if referer_hostname != request.get_host():
+            build_failure.REFERER = 'OTHER'
+    except KeyError:
+        # There is no referer
+        build_failure.REFERER = 'NO_REFERER'
+    build_failure.save()
 
     context = {'detail' : build_failure, 'error_types' : ErrorType }
 
-- 
2.7.4