* [error-report-web][PATCH 1/1] purge.py: Create script that will purge database
2018-01-24 21:44 [error-report-web][PATCH 0/1] purge.py: Create script that will purge database Amanda Brindle
@ 2018-01-24 21:44 ` Amanda Brindle
0 siblings, 0 replies; 2+ messages in thread
From: Amanda Brindle @ 2018-01-24 21:44 UTC (permalink / raw)
To: yocto; +Cc: paul.eggleton, Amanda Brindle
The script will be run on a regular basis to get rid of old reports that
we don't need. This will improve the performance of the application
since the database has grown to a huge size.
The script will remove reports older than thirty days if they have not
been referred to by a host other than the Error Reporting Tool.
The function details() in views.py will keep trick of the referer when
a build failure report is accessed. If there is no referer, we will note
that in the database in order to determine how often there is no
referer. It's possible that crawlers will access reports with no
referer; if that's the case, a lack of a referer should not determine
whether we save an older report or not.
Fixes [YOCTO #12332]
Signed-off-by: Amanda Brindle <amanda.r.brindle@intel.com>
---
Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++
Post/models.py | 10 ++++++++++
Post/purge.py | 29 ++++++++++++++++++++++++++++
Post/views.py | 16 +++++++++++++--
4 files changed, 72 insertions(+), 2 deletions(-)
create mode 100644 Post/migrations/0006_buildfailure_referer.py
create mode 100644 Post/purge.py
diff --git a/Post/migrations/0006_buildfailure_referer.py b/Post/migrations/0006_buildfailure_referer.py
new file mode 100644
index 0000000..5fad048
--- /dev/null
+++ b/Post/migrations/0006_buildfailure_referer.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('Post', '0005_build_error_type'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='buildfailure',
+ name='REFERER',
+ field=models.CharField(default=b'NOT_VISITED', max_length=14, choices=[(b'NO_REFERER', b'no_referer'), (b'OTHER', b'other'), (b'NOT_VISITED', b'not_visited')]),
+ ),
+ ]
diff --git a/Post/models.py b/Post/models.py
index ddf2fc7..fcf53ca 100644
--- a/Post/models.py
+++ b/Post/models.py
@@ -59,6 +59,16 @@ class BuildFailure(models.Model):
ERROR_DETAILS = models.TextField(max_length=int(settings.MAX_UPLOAD_SIZE))
BUILD = models.ForeignKey(Build)
LEV_DISTANCE = models.IntegerField(blank=True, null=True)
+ REFERER_CHOICES = (
+ ('NO_REFERER', 'no_referer'),
+ ('OTHER', 'other'),
+ ('NOT_VISITED', 'not_visited')
+ )
+ REFERER = models.CharField(
+ max_length = 14,
+ choices = REFERER_CHOICES,
+ default = 'NOT_VISITED'
+ )
def get_similar_fails(self):
if self.LEV_DISTANCE is None:
diff --git a/Post/purge.py b/Post/purge.py
new file mode 100644
index 0000000..829fada
--- /dev/null
+++ b/Post/purge.py
@@ -0,0 +1,29 @@
+from datetime import datetime
+from django.utils import timezone
+import os
+import sys
+
+def setup_django():
+ import django
+ # Get access to our Django model
+ newpath = os.path.abspath(os.path.dirname(__file__)) + '/..'
+ sys.path.append(newpath)
+ os.environ['DJANGO_SETTINGS_MODULE'] = 'project.settings'
+ django.setup()
+
+def main():
+ setup_django()
+ from Post.models import BuildFailure
+ items = BuildFailure.objects.all()
+ now = timezone.now()
+ for item in items:
+ if item.REFERER == 'OTHER' or item.REFERER == 'NO_REFERER':
+ continue
+ difference = now - item.BUILD.DATE
+ if difference.days > 30:
+ item.delete()
+
+if __name__ == "__main__":
+ main()
+
+
diff --git a/Post/views.py b/Post/views.py
index 7f2cffb..2ea94a7 100644
--- a/Post/views.py
+++ b/Post/views.py
@@ -22,6 +22,7 @@ from django.http import JsonResponse
from django.db.models import Q
import json
import urllib
+from urlparse import urlparse
class results_mode(object):
LATEST = 0
@@ -256,9 +257,20 @@ def search(request, mode=results_mode.LATEST, **kwargs):
def details(request, fail_id):
try:
- build_failure = BuildFailure.objects.get(id=fail_id)
+ build_failure = BuildFailure.objects.get(id=fail_id)
except ObjectDoesNotExist:
- build_failure = None
+ build_failure = None
+ try:
+ referer = urlparse(request.META['HTTP_REFERER'])
+ referer_hostname = referer.hostname
+ if referer.port:
+ referer_hostname += ":" + str(referer.port)
+ if referer_hostname != request.get_host():
+ build_failure.REFERER = 'OTHER'
+ except KeyError:
+ # There is no referer
+ build_failure.REFERER = 'NO_REFERER'
+ build_failure.save()
context = {'detail' : build_failure, 'error_types' : ErrorType }
--
2.7.4
^ permalink raw reply related [flat|nested] 2+ messages in thread