diff --git a/requirements.txt b/requirements.txt index 357b6fe30..44bd810d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,10 +28,10 @@ decorator==5.1.1 defusedxml==0.7.1 distro==1.7.0 Django==4.2.22 +django-altcha==0.2.0 django-crispy-forms==2.3 django-environ==0.11.2 django-filter==24.3 -django-recaptcha==4.0.0 django-widget-tweaks==1.5.0 djangorestframework==3.15.2 doc8==0.11.1 diff --git a/setup.cfg b/setup.cfg index da5c028ef..f6b529e8c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,6 +65,7 @@ install_requires = crispy-bootstrap4>=2024.1 django-environ>=0.11.0 gunicorn>=23.0.0 + django-altcha==0.2.0 # for the API doc drf-spectacular[sidecar]>=0.24.2 @@ -101,8 +102,6 @@ install_requires = python-dotenv texttable - django-recaptcha>=4.0.0 - [options.extras_require] dev = diff --git a/vulnerabilities/admin.py b/vulnerabilities/admin.py index 176e3c0c0..558cd3f5d 100644 --- a/vulnerabilities/admin.py +++ b/vulnerabilities/admin.py @@ -21,6 +21,10 @@ from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilitySeverity +admin.site.site_header = "VulnerableCode Administration" +admin.site.site_title = "VulnerableCode Admin Portal" +admin.site.index_title = "Welcome to VulnerableCode Management" + @admin.register(Vulnerability) class VulnerabilityAdmin(admin.ModelAdmin): @@ -125,6 +129,7 @@ def __init__(self, *args, **kwargs): def save(self, commit=True): group = super().save(commit=commit) + group.save() self.save_m2m() group.user_set.set(self.cleaned_data["users"]) return group diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index e9f967b79..d5bae0914 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -769,7 +769,7 @@ def has_permission(self, request, view): class PipelineRunAPISerializer(serializers.HyperlinkedModelSerializer): status = serializers.SerializerMethodField() - execution_time = serializers.SerializerMethodField() + runtime = serializers.SerializerMethodField() log = serializers.SerializerMethodField() class Meta: @@ -777,7 +777,7 @@ class Meta: fields = [ "run_id", "status", - "execution_time", + "runtime", "run_start_date", "run_end_date", "run_exitcode", @@ -791,9 +791,9 @@ class Meta: def get_status(self, run): return run.status - def get_execution_time(self, run): - if run.execution_time: - return round(run.execution_time, 2) + def get_runtime(self, run): + if run.runtime: + return f"{round(run.runtime, 2)}s" def get_log(self, run): """Return only last 5000 character of log.""" @@ -802,7 +802,8 @@ def get_log(self, run): class PipelineScheduleAPISerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( - view_name="schedule-detail", lookup_field="pipeline_id" + view_name="pipelines-detail", + lookup_field="pipeline_id", ) latest_run = serializers.SerializerMethodField() next_run_date = serializers.SerializerMethodField() @@ -830,6 +831,12 @@ def get_latest_run(self, schedule): return PipelineRunAPISerializer(latest).data return None + def to_representation(self, schedule): + representation = super().to_representation(schedule) + representation["run_interval"] = f"{schedule.run_interval}hr" + representation["execution_timeout"] = f"{schedule.execution_timeout}hr" + return representation + class PipelineScheduleCreateSerializer(serializers.ModelSerializer): class Meta: @@ -883,6 +890,11 @@ def get_permissions(self): return [IsAdminWithSessionAuth()] return super().get_permissions() + def get_view_name(self): + if self.detail: + return "Pipeline Instance" + return "Pipeline Jobs" + class AdvisoriesPackageV2ViewSet(viewsets.ReadOnlyModelViewSet): queryset = PackageV2.objects.all().prefetch_related( diff --git a/vulnerabilities/forms.py b/vulnerabilities/forms.py index 7d955ac37..7ee348354 100644 --- a/vulnerabilities/forms.py +++ b/vulnerabilities/forms.py @@ -10,8 +10,7 @@ from django import forms from django.contrib.admin.forms import AdminAuthenticationForm from django.core.validators import validate_email -from django_recaptcha.fields import ReCaptchaField -from django_recaptcha.widgets import ReCaptchaV2Checkbox +from django_altcha import AltchaField from vulnerabilities.models import ApiUser @@ -45,12 +44,12 @@ class AdvisorySearchForm(forms.Form): class ApiUserCreationForm(forms.ModelForm): - """ - Support a simplified creation for API-only users directly from the UI. - """ + """Support a simplified creation for API-only users directly from the UI.""" - captcha = ReCaptchaField( - error_messages={"required": ("Captcha is required")}, widget=ReCaptchaV2Checkbox + captcha = AltchaField( + floating=True, + hidefooter=True, + hidelogo=True, ) class Meta: @@ -110,9 +109,8 @@ class PipelineSchedulePackageForm(forms.Form): class AdminLoginForm(AdminAuthenticationForm): - captcha = ReCaptchaField( - error_messages={ - "required": ("Captcha is required"), - }, - widget=ReCaptchaV2Checkbox(), + captcha = AltchaField( + floating=True, + hidefooter=True, + hidelogo=True, ) diff --git a/vulnerabilities/migrations/0096_alter_pipelineschedule_run_interval.py b/vulnerabilities/migrations/0096_alter_pipelineschedule_run_interval.py new file mode 100644 index 000000000..f641ddd95 --- /dev/null +++ b/vulnerabilities/migrations/0096_alter_pipelineschedule_run_interval.py @@ -0,0 +1,71 @@ +# Generated by Django 4.2.22 on 2025-06-24 16:33 + +import django.core.validators +from django.db import migrations, models +from datetime import datetime +from datetime import timezone + +from aboutcode.pipeline import LoopProgress + +class Migration(migrations.Migration): + def update_old_interval(apps, schema_editor): + Pipeline = apps.get_model("vulnerabilities", "PipelineSchedule") + pipeline_schedule = Pipeline.objects.all() + + log(f"\nUpdating interval for {pipeline_schedule.count():,d} pipeline") + progress = LoopProgress( + total_iterations=pipeline_schedule.count(), + progress_step=10, + logger=log, + ) + for pipeline in progress.iter(pipeline_schedule.iterator()): + pipeline.run_interval = pipeline.run_interval * 24 + pipeline.save() + + def reverse_update_old_interval(apps, schema_editor): + Pipeline = apps.get_model("vulnerabilities", "PipelineSchedule") + pipeline_schedule = Pipeline.objects.all() + + log(f"\nReverse interval for {pipeline_schedule.count():,d} pipeline") + progress = LoopProgress( + total_iterations=pipeline_schedule.count(), + progress_step=10, + logger=log, + ) + for pipeline in progress.iter(pipeline_schedule.iterator()): + pipeline.run_interval = pipeline.run_interval / 24 + pipeline.save() + + dependencies = [ + ("vulnerabilities", "0095_alter_apiuser_options"), + ] + + operations = [ + migrations.AlterField( + model_name="pipelineschedule", + name="run_interval", + field=models.PositiveSmallIntegerField( + default=24, + help_text="Number of hours to wait between run of this pipeline.", + validators=[ + django.core.validators.MinValueValidator( + 1, message="Interval must be at least 1 hour." + ), + django.core.validators.MaxValueValidator( + 8760, message="Interval must be at most 8760 hours." + ), + ], + ), + ), + migrations.RunPython( + code=update_old_interval, + reverse_code=reverse_update_old_interval, + ), + ] + + +def log(message): + now_local = datetime.now(timezone.utc).astimezone() + timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + message = f"{timestamp} {message}" + print(message) \ No newline at end of file diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e1c656e2b..77781b055 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2023,6 +2023,7 @@ class Meta: class Status(models.TextChoices): UNKNOWN = "unknown" + DISABLED = "disabled" RUNNING = "running" SUCCESS = "success" FAILURE = "failure" @@ -2115,7 +2116,7 @@ def run_running(self): return self.status == self.Status.RUNNING @property - def execution_time(self): + def runtime(self): """Return the pipeline execution time.""" if not self.run_start_date or (not self.run_end_date and not self.run_running): return @@ -2278,11 +2279,11 @@ class PipelineSchedule(models.Model): run_interval = models.PositiveSmallIntegerField( validators=[ - MinValueValidator(1, message="Interval must be at least 1 day."), - MaxValueValidator(365, message="Interval must be at most 365 days."), + MinValueValidator(1, message="Interval must be at least 1 hour."), + MaxValueValidator(8760, message="Interval must be at most 8760 hours."), ], - default=1, - help_text=("Number of days to wait between run of this pipeline."), + default=24, + help_text=("Number of hours to wait between run of this pipeline."), ) schedule_work_id = models.CharField( @@ -2360,6 +2361,13 @@ def latest_run_date(self): latest_run = self.pipelineruns.values("run_start_date").first() return latest_run["run_start_date"] + @property + def latest_run_end_date(self): + if not self.pipelineruns.exists(): + return + latest_run = self.pipelineruns.values("run_end_date").first() + return latest_run["run_end_date"] + @property def next_run_date(self): if not self.is_active: @@ -2367,7 +2375,7 @@ def next_run_date(self): current_date_time = datetime.datetime.now(tz=datetime.timezone.utc) if self.latest_run_date: - next_execution = self.latest_run_date + datetime.timedelta(days=self.run_interval) + next_execution = self.latest_run_date + datetime.timedelta(hours=self.run_interval) if next_execution > current_date_time: return next_execution @@ -2376,7 +2384,7 @@ def next_run_date(self): @property def status(self): if not self.is_active: - return + return PipelineRun.Status.DISABLED if self.pipelineruns.exists(): latest = self.pipelineruns.only("pk").first() diff --git a/vulnerabilities/schedules.py b/vulnerabilities/schedules.py index 1c34bd759..2c2e5366b 100644 --- a/vulnerabilities/schedules.py +++ b/vulnerabilities/schedules.py @@ -14,7 +14,6 @@ from redis.exceptions import ConnectionError from vulnerabilities.tasks import enqueue_pipeline -from vulnerablecode.settings import VULNERABLECODE_PIPELINE_TIMEOUT log = logging.getLogger(__name__) scheduler = django_rq.get_scheduler() @@ -29,14 +28,13 @@ def schedule_execution(pipeline_schedule, execute_now=False): if not execute_now: first_execution = pipeline_schedule.next_run_date - interval_in_seconds = pipeline_schedule.run_interval * 24 * 60 * 60 + interval_in_seconds = pipeline_schedule.run_interval * 60 * 60 job = scheduler.schedule( scheduled_time=first_execution, func=enqueue_pipeline, args=[pipeline_schedule.pipeline_id], interval=interval_in_seconds, - result_ttl=f"{VULNERABLECODE_PIPELINE_TIMEOUT}h", repeat=None, ) return job._id diff --git a/vulnerabilities/tasks.py b/vulnerabilities/tasks.py index 07e0c99ac..e035985a2 100644 --- a/vulnerabilities/tasks.py +++ b/vulnerabilities/tasks.py @@ -106,6 +106,18 @@ def set_run_failure(job, connection, type, value, traceback): def enqueue_pipeline(pipeline_id): pipeline_schedule = models.PipelineSchedule.objects.get(pipeline_id=pipeline_id) + if pipeline_schedule.status in [ + models.PipelineRun.Status.RUNNING, + models.PipelineRun.Status.QUEUED, + ]: + logger.warning( + ( + f"Cannot enqueue a new execution for {pipeline_id} " + "until the previous one has finished." + ) + ) + return + run = models.PipelineRun.objects.create( pipeline=pipeline_schedule, ) diff --git a/vulnerabilities/templates/admin_login.html b/vulnerabilities/templates/admin_login.html index 2fa9dc484..a656ab058 100644 --- a/vulnerabilities/templates/admin_login.html +++ b/vulnerabilities/templates/admin_login.html @@ -3,6 +3,38 @@ {% block extrastyle %}{{ block.super }} {{ form.media }} + + {% endblock %} {% block bodyclass %}{{ block.super }} login{% endblock %} @@ -57,10 +89,8 @@ {% translate 'Forgotten your password or username?' %} {% endif %} -
-
- {{ form.captcha }} -
+
+ {{ form.captcha }}
diff --git a/vulnerabilities/templates/api_user_creation_form.html b/vulnerabilities/templates/api_user_creation_form.html index 4c596f094..5a913f5ae 100644 --- a/vulnerabilities/templates/api_user_creation_form.html +++ b/vulnerabilities/templates/api_user_creation_form.html @@ -26,11 +26,13 @@
{% endif %}
-

- VulnerableCode API key request -

-
+
+

VulnerableCode API Key Request

+
+
+ +

You need an API key to access the VulnerableCode JSON REST API. Please check the live OpenAPI documentation @@ -41,17 +43,34 @@


-
- {% csrf_token %} - {% for field in form %} -
- -
- {{ field }} -
+
+
+ + {% csrf_token %} +
+
+
- {% endfor %} - - +
+
+
+ +
+
+
+
+ +
+
+
+ {{ form.captcha }} +
+ + +
+
{% endblock %} diff --git a/vulnerabilities/templates/includes/job_status.html b/vulnerabilities/templates/includes/job_status.html index 192da5281..ca2d35b14 100644 --- a/vulnerabilities/templates/includes/job_status.html +++ b/vulnerabilities/templates/includes/job_status.html @@ -40,6 +40,13 @@ Stale {% endif %} +{% elif status == "disabled" %} + {% if compact %} + Disabled + {% else %} + + Disabled + {% endif %} {% else %} {% if compact %} Unknown diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index 51b796ae4..b7a7fba0a 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -20,8 +20,8 @@
- - Pipeline Schedule + + Pipeline Dashboard API diff --git a/vulnerabilities/templates/pipeline_schedule_list.html b/vulnerabilities/templates/pipeline_dashboard.html similarity index 86% rename from vulnerabilities/templates/pipeline_schedule_list.html rename to vulnerabilities/templates/pipeline_dashboard.html index 12d117117..d8b03a91a 100644 --- a/vulnerabilities/templates/pipeline_schedule_list.html +++ b/vulnerabilities/templates/pipeline_dashboard.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% block title %} -Pipeline Schedule +Pipeline Dashboard {% endblock %} {% block extrahead %} @@ -33,7 +33,7 @@
-

Pipeline Schedule

+

Pipeline Dashboard

@@ -46,7 +46,14 @@

Pipeline Schedule

+
+
+

+ {{ active_pipeline_count|default:0 }} active pipeline{{ active_pipeline_count|default:0|pluralize }}, + {{ disabled_pipeline_count|default:0 }} disabled pipeline{{ disabled_pipeline_count|default:0|pluralize }} +

+
@@ -57,8 +64,8 @@

Pipeline Schedule

Active
Interval
Status
-
Latest Run
-
Next Run
+
Last Run End Time
+
Next Run Start
@@ -72,22 +79,22 @@

Pipeline Schedule

{{ schedule.pipeline_id }}
{{ schedule.is_active|yesno:"Yes,No" }}
-
{{ schedule.run_interval }}day
+
+ {{ schedule.run_interval }} hour{{ schedule.run_interval|pluralize }} +
{% include "includes/job_status.html" with status=schedule.status %}
-
- {% if schedule.latest_run_date %} - {{ schedule.latest_run_date|date:"Y-m-d h:i a T" }} +
+ {% if schedule.latest_run_end_date %} + {{ schedule.latest_run_end_date|date:"Y-m-d h:i a T" }} {% else %} N/A {% endif %}
-
+
{{ schedule.next_run_date|date:"Y-m-d" }}
diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html index 6ebf92807..881d56c95 100644 --- a/vulnerabilities/templates/pipeline_run_details.html +++ b/vulnerabilities/templates/pipeline_run_details.html @@ -34,8 +34,10 @@ {% endblock %} {% block content %} -
+ + Back to All Runs +

{{ pipeline_name }} Run Log

@@ -43,7 +45,7 @@

{{ pipeline_name }} Run Log

Pipeline ID

-

+

{{ run.pipeline.pipeline_id }} {{ pipeline_name }} Run Log

-

Execution Time

+

Runtime

- {% if run.execution_time %} - {{ run.execution_time|humanize_duration }} + {% if run.runtime %} + {{ run.runtime|humanize_duration }} {% else %} N/A {% endif %} @@ -114,7 +116,7 @@

{{ pipeline_name }} Run Log

Job ID

-

+

{{ run.run_id }} Log Output

{% endif %} - - - Back to All Runs - + {% if run.run_output or run.log %} + + Back to All Runs + + {% endif %}
-
{% endblock %} diff --git a/vulnerabilities/templates/pipeline_run_list.html b/vulnerabilities/templates/pipeline_run_list.html index f71bb94da..88b595ef8 100644 --- a/vulnerabilities/templates/pipeline_run_list.html +++ b/vulnerabilities/templates/pipeline_run_list.html @@ -34,6 +34,7 @@
+ Back to Dashboard

{{ pipeline_name }} Runs

@@ -56,7 +57,7 @@

{{ pipeline_name }} Runs

Job ID
Status
-
Execution Time
+
Runtime
Start Time
End Time
@@ -78,8 +79,8 @@

{{ pipeline_name }} Runs

- {% if run.execution_time %} - {{ run.execution_time|humanize_duration }} + {% if run.runtime %} + {{ run.runtime|humanize_duration }} {% else %} N/A {% endif %} @@ -109,7 +110,9 @@

{{ pipeline_name }} Runs

{% endfor %}
- Back to Schedules + {% if run_list|length > 10 %} + Back to Dashboard + {% endif %}
{% if is_paginated %}