From b885f5d0e9105e3d4848b4a8cc06de74734be870 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:00:24 +0200 Subject: [PATCH 01/13] Add generic_forms app with Form, FormQuestion and FormAnswer models Foundation for the generic form system: conference-scoped forms with admin-authorable questions and JSON answers (versioned envelope). DB constraints: one answer per (form, user); at most one form per (conference, purpose) except for generic-purpose forms. --- backend/generic_forms/__init__.py | 0 backend/generic_forms/apps.py | 6 ++ .../generic_forms/migrations/0001_initial.py | 70 +++++++++++++++ backend/generic_forms/migrations/__init__.py | 0 backend/generic_forms/models.py | 90 +++++++++++++++++++ backend/generic_forms/tests/__init__.py | 0 backend/generic_forms/tests/factories.py | 35 ++++++++ backend/generic_forms/tests/test_models.py | 39 ++++++++ backend/pycon/settings/base.py | 1 + 9 files changed, 241 insertions(+) create mode 100644 backend/generic_forms/__init__.py create mode 100644 backend/generic_forms/apps.py create mode 100644 backend/generic_forms/migrations/0001_initial.py create mode 100644 backend/generic_forms/migrations/__init__.py create mode 100644 backend/generic_forms/models.py create mode 100644 backend/generic_forms/tests/__init__.py create mode 100644 backend/generic_forms/tests/factories.py create mode 100644 backend/generic_forms/tests/test_models.py diff --git a/backend/generic_forms/__init__.py b/backend/generic_forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/generic_forms/apps.py b/backend/generic_forms/apps.py new file mode 100644 index 0000000000..a819ae21d2 --- /dev/null +++ b/backend/generic_forms/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class GenericFormsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "generic_forms" diff --git a/backend/generic_forms/migrations/0001_initial.py b/backend/generic_forms/migrations/0001_initial.py new file mode 100644 index 0000000000..fef6cd1cf7 --- /dev/null +++ b/backend/generic_forms/migrations/0001_initial.py @@ -0,0 +1,70 @@ +# Generated by Django 5.2.8 on 2026-08-06 12:56 + +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('conferences', '0058_conference_hostname'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Form', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('purpose', models.CharField(choices=[('grant', 'Grant'), ('generic', 'Generic')], max_length=32)), + ('name', models.CharField(max_length=200)), + ('conference', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='forms', to='conferences.conference')), + ], + ), + migrations.CreateModel( + name='FormAnswer', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('answers', models.JSONField(default=dict)), + ('form', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='answers', to='generic_forms.form')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='form_answers', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='FormQuestion', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('label', models.CharField(max_length=300)), + ('description', models.TextField(blank=True)), + ('question_type', models.CharField(choices=[('text', 'Text'), ('textarea', 'Textarea'), ('select', 'Select'), ('multi_select', 'Multi select'), ('boolean', 'Boolean'), ('url', 'URL')], max_length=32)), + ('options', models.JSONField(blank=True, default=list)), + ('required', models.BooleanField(default=False)), + ('max_length', models.PositiveIntegerField(blank=True, null=True)), + ('order', models.PositiveIntegerField(default=0)), + ('active', models.BooleanField(default=True)), + ('form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='questions', to='generic_forms.form')), + ], + options={ + 'ordering': ['order', 'id'], + }, + ), + migrations.AddConstraint( + model_name='form', + constraint=models.UniqueConstraint(condition=models.Q(('purpose', 'generic'), _negated=True), fields=('conference', 'purpose'), name='unique_form_per_conference_and_purpose'), + ), + migrations.AddConstraint( + model_name='formanswer', + constraint=models.UniqueConstraint(fields=('form', 'user'), name='unique_form_answer_per_user'), + ), + ] diff --git a/backend/generic_forms/migrations/__init__.py b/backend/generic_forms/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/generic_forms/models.py b/backend/generic_forms/models.py new file mode 100644 index 0000000000..ae30762154 --- /dev/null +++ b/backend/generic_forms/models.py @@ -0,0 +1,90 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ +from model_utils.models import TimeStampedModel + +from users.models import User + + +class Form(TimeStampedModel): + class Purpose(models.TextChoices): + GRANT = "grant", _("Grant") + GENERIC = "generic", _("Generic") + + conference = models.ForeignKey( + "conferences.Conference", + on_delete=models.CASCADE, + related_name="forms", + ) + purpose = models.CharField(max_length=32, choices=Purpose.choices) + name = models.CharField(max_length=200) + + def __str__(self): + return f"{self.name} ({self.purpose}, {self.conference.name})" + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["conference", "purpose"], + condition=~models.Q(purpose="generic"), + name="unique_form_per_conference_and_purpose", + ) + ] + + +class FormQuestion(TimeStampedModel): + class QuestionType(models.TextChoices): + TEXT = "text", _("Text") + TEXTAREA = "textarea", _("Textarea") + SELECT = "select", _("Select") + MULTI_SELECT = "multi_select", _("Multi select") + BOOLEAN = "boolean", _("Boolean") + URL = "url", _("URL") + + form = models.ForeignKey( + Form, + on_delete=models.CASCADE, + related_name="questions", + ) + label = models.CharField(max_length=300) + description = models.TextField(blank=True) + question_type = models.CharField(max_length=32, choices=QuestionType.choices) + # list of {"id": "vegan", "label": "Vegan"}; only for select/multi_select + options = models.JSONField(blank=True, default=list) + required = models.BooleanField(default=False) + max_length = models.PositiveIntegerField(null=True, blank=True) + order = models.PositiveIntegerField(default=0) + active = models.BooleanField(default=True) + + def __str__(self): + return self.label + + class Meta: + ordering = ["order", "id"] + + +class FormAnswer(TimeStampedModel): + form = models.ForeignKey( + Form, + on_delete=models.PROTECT, + related_name="answers", + ) + user = models.ForeignKey( + User, + on_delete=models.CASCADE, + related_name="form_answers", + ) + # versioned envelope: {"version": 1, "answers": {"": value}} + # value types (version 1): text/textarea/url -> str, select -> option id, + # multi_select -> list of option ids, boolean -> bool + answers = models.JSONField(default=dict) + + def __str__(self): + return f"Answers of {self.user_id} to {self.form_id}" + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["form", "user"], + name="unique_form_answer_per_user", + ) + ] diff --git a/backend/generic_forms/tests/__init__.py b/backend/generic_forms/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/generic_forms/tests/factories.py b/backend/generic_forms/tests/factories.py new file mode 100644 index 0000000000..93c32c2010 --- /dev/null +++ b/backend/generic_forms/tests/factories.py @@ -0,0 +1,35 @@ +import factory +from factory.django import DjangoModelFactory + +from conferences.tests.factories import ConferenceFactory +from generic_forms.models import Form, FormAnswer, FormQuestion +from users.tests.factories import UserFactory + + +class FormFactory(DjangoModelFactory): + class Meta: + model = Form + + conference = factory.SubFactory(ConferenceFactory) + purpose = Form.Purpose.GENERIC + name = factory.Faker("sentence", nb_words=3) + + +class FormQuestionFactory(DjangoModelFactory): + class Meta: + model = FormQuestion + + form = factory.SubFactory(FormFactory) + label = factory.Faker("sentence", nb_words=5) + question_type = FormQuestion.QuestionType.TEXT + required = False + order = factory.Sequence(lambda n: n) + + +class FormAnswerFactory(DjangoModelFactory): + class Meta: + model = FormAnswer + + form = factory.SubFactory(FormFactory) + user = factory.SubFactory(UserFactory) + answers = factory.LazyFunction(dict) diff --git a/backend/generic_forms/tests/test_models.py b/backend/generic_forms/tests/test_models.py new file mode 100644 index 0000000000..56ede7130a --- /dev/null +++ b/backend/generic_forms/tests/test_models.py @@ -0,0 +1,39 @@ +import pytest +from django.db import IntegrityError + +from generic_forms.models import Form +from generic_forms.tests.factories import FormAnswerFactory, FormFactory + +pytestmark = pytest.mark.django_db + + +def test_form_answer_is_unique_per_form_and_user(): + answer = FormAnswerFactory() + + with pytest.raises(IntegrityError): + FormAnswerFactory(form=answer.form, user=answer.user) + + +def test_same_user_can_answer_different_forms(): + answer = FormAnswerFactory() + + FormAnswerFactory(user=answer.user) + + +def test_only_one_form_per_conference_and_purpose(): + form = FormFactory(purpose=Form.Purpose.GRANT) + + with pytest.raises(IntegrityError): + FormFactory(conference=form.conference, purpose=Form.Purpose.GRANT) + + +def test_multiple_generic_forms_per_conference_are_allowed(): + form = FormFactory(purpose=Form.Purpose.GENERIC) + + FormFactory(conference=form.conference, purpose=Form.Purpose.GENERIC) + + +def test_same_purpose_is_allowed_on_different_conferences(): + FormFactory(purpose=Form.Purpose.GRANT) + + FormFactory(purpose=Form.Purpose.GRANT) diff --git a/backend/pycon/settings/base.py b/backend/pycon/settings/base.py index 883498df30..8293a156bb 100644 --- a/backend/pycon/settings/base.py +++ b/backend/pycon/settings/base.py @@ -128,6 +128,7 @@ "billing.apps.BillingConfig", "privacy_policy.apps.PrivacyPolicyConfig", "visa.apps.VisaConfig", + "generic_forms.apps.GenericFormsConfig", ] MIDDLEWARE = [ From 3dfe2ee7674bd4111d6b76de6bfed121616d3603 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:01:48 +0200 Subject: [PATCH 02/13] Freeze semantic question fields once a form has answers question_type, options, required and the parent form become immutable (and deletion is blocked) as soon as any FormAnswer exists, so stored answers always match their questions. Label, description, order and active stay editable; deactivation replaces deletion. --- backend/generic_forms/models.py | 40 ++++++++++ backend/generic_forms/tests/test_models.py | 93 +++++++++++++++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/backend/generic_forms/models.py b/backend/generic_forms/models.py index ae30762154..f4272bf9b0 100644 --- a/backend/generic_forms/models.py +++ b/backend/generic_forms/models.py @@ -1,3 +1,4 @@ +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ from model_utils.models import TimeStampedModel @@ -55,6 +56,45 @@ class QuestionType(models.TextChoices): order = models.PositiveIntegerField(default=0) active = models.BooleanField(default=True) + # Fields that define what an answer means; frozen once any answer exists + # so stored answers always match their questions. label/description/order/ + # active stay editable (deactivate instead of delete). + FROZEN_FIELDS = ("form_id", "question_type", "options", "required") + + def clean(self): + super().clean() + self._check_frozen_fields() + + def save(self, *args, **kwargs): + self._check_frozen_fields() + super().save(*args, **kwargs) + + def delete(self, *args, **kwargs): + if self.form.answers.exists(): + raise ValidationError( + "This question cannot be deleted because the form already has " + "answers. Deactivate it instead." + ) + return super().delete(*args, **kwargs) + + def _check_frozen_fields(self): + if not self.pk: + return + + stored = FormQuestion.objects.get(pk=self.pk) + changed = [ + field + for field in self.FROZEN_FIELDS + if getattr(stored, field) != getattr(self, field) + ] + if changed and stored.form.answers.exists(): + fields = ", ".join(field.removesuffix("_id") for field in changed) + raise ValidationError( + f"The form already has answers; these fields cannot be " + f"changed: {fields}. Add a new question or deactivate this " + f"one instead." + ) + def __str__(self): return self.label diff --git a/backend/generic_forms/tests/test_models.py b/backend/generic_forms/tests/test_models.py index 56ede7130a..de3391b195 100644 --- a/backend/generic_forms/tests/test_models.py +++ b/backend/generic_forms/tests/test_models.py @@ -1,8 +1,13 @@ import pytest +from django.core.exceptions import ValidationError from django.db import IntegrityError -from generic_forms.models import Form -from generic_forms.tests.factories import FormAnswerFactory, FormFactory +from generic_forms.models import Form, FormQuestion +from generic_forms.tests.factories import ( + FormAnswerFactory, + FormFactory, + FormQuestionFactory, +) pytestmark = pytest.mark.django_db @@ -37,3 +42,87 @@ def test_same_purpose_is_allowed_on_different_conferences(): FormFactory(purpose=Form.Purpose.GRANT) FormFactory(purpose=Form.Purpose.GRANT) + + +def _answered_question(**kwargs): + question = FormQuestionFactory(**kwargs) + FormAnswerFactory(form=question.form) + return question + + +def test_question_type_is_frozen_once_form_has_answers(): + question = _answered_question(question_type=FormQuestion.QuestionType.TEXT) + + question.question_type = FormQuestion.QuestionType.TEXTAREA + with pytest.raises(ValidationError, match="question_type"): + question.save() + + +def test_options_are_frozen_once_form_has_answers(): + question = _answered_question( + question_type=FormQuestion.QuestionType.SELECT, + options=[{"id": "a", "label": "A"}], + ) + + question.options = [{"id": "b", "label": "B"}] + with pytest.raises(ValidationError, match="options"): + question.save() + + +def test_required_is_frozen_once_form_has_answers(): + question = _answered_question(required=False) + + question.required = True + with pytest.raises(ValidationError, match="required"): + question.save() + + +def test_question_cannot_move_to_another_form_once_answered(): + question = _answered_question() + + question.form = FormFactory() + with pytest.raises(ValidationError, match="form"): + question.save() + + +def test_semantic_fields_are_editable_while_form_has_no_answers(): + question = FormQuestionFactory(question_type=FormQuestion.QuestionType.TEXT) + + question.question_type = FormQuestion.QuestionType.TEXTAREA + question.required = True + question.save() + + +def test_label_description_order_active_stay_editable_once_answered(): + question = _answered_question() + + question.label = "Updated label" + question.description = "Updated description" + question.order = 42 + question.active = False + question.save() + + question.refresh_from_db() + assert question.label == "Updated label" + assert question.active is False + + +def test_question_cannot_be_deleted_once_form_has_answers(): + question = _answered_question() + + with pytest.raises(ValidationError, match="deleted"): + question.delete() + + +def test_question_can_be_deleted_while_form_has_no_answers(): + question = FormQuestionFactory() + + question.delete() + + assert not FormQuestion.objects.filter(pk=question.pk).exists() + + +def test_new_question_can_be_added_to_answered_form(): + question = _answered_question() + + FormQuestionFactory(form=question.form) From 43bb587d5c4c94622714c6fa85cb390c2327fd76 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:03:24 +0200 Subject: [PATCH 03/13] Add validate_answers service and versioned answers envelope validate_answers checks a flat {question_id: value} map against a form's active questions: required, per-type value shape, option membership (every multi-select item), URL format and max length. wrap_answers/unwrap_answers implement the {version: 1, answers: {...}} storage envelope with version dispatch. --- backend/generic_forms/services.py | 88 +++++++++ backend/generic_forms/tests/test_services.py | 181 +++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 backend/generic_forms/services.py create mode 100644 backend/generic_forms/tests/test_services.py diff --git a/backend/generic_forms/services.py b/backend/generic_forms/services.py new file mode 100644 index 0000000000..a37598514a --- /dev/null +++ b/backend/generic_forms/services.py @@ -0,0 +1,88 @@ +from collections import defaultdict + +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator + +from generic_forms.models import Form, FormQuestion + +ANSWERS_VERSION = 1 + + +def wrap_answers(answers: dict) -> dict: + return {"version": ANSWERS_VERSION, "answers": answers} + + +def unwrap_answers(envelope: dict) -> dict: + version = envelope.get("version") + if version != ANSWERS_VERSION: + raise ValueError(f"Unknown answers version: {version}") + return envelope["answers"] + + +def validate_answers(form: Form, answers: dict) -> dict[str, list[str]]: + """Validate a flat {question_id: value} map against the form's active + questions. Returns {question_id: [error messages]}; empty dict when valid. + """ + errors: dict[str, list[str]] = defaultdict(list) + questions = { + str(question.pk): question for question in form.questions.filter(active=True) + } + + for question_id in answers: + if question_id not in questions: + errors[question_id].append("Unknown or inactive question.") + + for question_id, question in questions.items(): + value = answers.get(question_id) + + if value is None or value == "" or value == []: + if question.required: + errors[question_id].append("This question is required.") + continue + + errors[question_id].extend(_validate_value(question, value)) + + return { + question_id: messages for question_id, messages in errors.items() if messages + } + + +def _validate_value(question: FormQuestion, value) -> list[str]: + question_type = question.question_type + types = FormQuestion.QuestionType + + if question_type in (types.TEXT, types.TEXTAREA, types.URL): + if not isinstance(value, str): + return ["Invalid value: expected text."] + if question.max_length and len(value) > question.max_length: + return [f"Cannot be longer than {question.max_length} characters."] + if question_type == types.URL: + try: + URLValidator()(value) + except ValidationError: + return ["Invalid URL."] + return [] + + if question_type == types.SELECT: + if not isinstance(value, str) or value not in _option_ids(question): + return ["Invalid option."] + return [] + + if question_type == types.MULTI_SELECT: + if not isinstance(value, list): + return ["Invalid value: expected a list of options."] + invalid = [item for item in value if item not in _option_ids(question)] + if invalid: + return ["Invalid options: " + ", ".join(map(str, invalid)) + "."] + return [] + + if question_type == types.BOOLEAN: + if not isinstance(value, bool): + return ["Invalid value: expected true or false."] + return [] + + return ["Unknown question type."] + + +def _option_ids(question: FormQuestion) -> set[str]: + return {option["id"] for option in question.options} diff --git a/backend/generic_forms/tests/test_services.py b/backend/generic_forms/tests/test_services.py new file mode 100644 index 0000000000..19ec0c82f1 --- /dev/null +++ b/backend/generic_forms/tests/test_services.py @@ -0,0 +1,181 @@ +import pytest + +from generic_forms.models import FormQuestion +from generic_forms.services import unwrap_answers, validate_answers, wrap_answers +from generic_forms.tests.factories import FormFactory, FormQuestionFactory + +pytestmark = pytest.mark.django_db + + +OPTIONS = [{"id": "vegan", "label": "Vegan"}, {"id": "veggie", "label": "Veggie"}] + + +def _question(question_type, **kwargs): + return FormQuestionFactory(question_type=question_type, **kwargs) + + +def test_valid_answers_return_no_errors(): + form = FormFactory() + text = _question(FormQuestion.QuestionType.TEXT, form=form, required=True) + textarea = _question(FormQuestion.QuestionType.TEXTAREA, form=form) + select = _question(FormQuestion.QuestionType.SELECT, form=form, options=OPTIONS) + multi = _question( + FormQuestion.QuestionType.MULTI_SELECT, form=form, options=OPTIONS + ) + boolean = _question(FormQuestion.QuestionType.BOOLEAN, form=form) + url = _question(FormQuestion.QuestionType.URL, form=form) + + errors = validate_answers( + form, + { + str(text.pk): "an answer", + str(textarea.pk): "a longer answer", + str(select.pk): "vegan", + str(multi.pk): ["vegan", "veggie"], + str(boolean.pk): True, + str(url.pk): "https://example.com", + }, + ) + + assert errors == {} + + +def test_missing_required_answer_is_an_error(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.TEXT, form=form, required=True) + + errors = validate_answers(form, {}) + + assert "required" in errors[str(question.pk)][0].lower() + + +def test_empty_string_fails_required(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.TEXT, form=form, required=True) + + errors = validate_answers(form, {str(question.pk): ""}) + + assert "required" in errors[str(question.pk)][0].lower() + + +def test_optional_question_can_be_omitted(): + form = FormFactory() + _question(FormQuestion.QuestionType.TEXT, form=form, required=False) + + assert validate_answers(form, {}) == {} + + +def test_answering_false_satisfies_a_required_boolean(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.BOOLEAN, form=form, required=True) + + assert validate_answers(form, {str(question.pk): False}) == {} + + +def test_unknown_question_id_is_an_error(): + form = FormFactory() + + errors = validate_answers(form, {"9999": "hello"}) + + assert "9999" in errors + + +def test_inactive_question_id_is_an_error(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.TEXT, form=form, active=False) + + errors = validate_answers(form, {str(question.pk): "hello"}) + + assert str(question.pk) in errors + + +def test_text_answer_must_be_a_string(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.TEXT, form=form) + + errors = validate_answers(form, {str(question.pk): 123}) + + assert str(question.pk) in errors + + +def test_text_answer_respects_max_length(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.TEXT, form=form, max_length=5) + + errors = validate_answers(form, {str(question.pk): "too long"}) + + assert "5" in errors[str(question.pk)][0] + + +def test_select_answer_must_be_a_known_option(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.SELECT, form=form, options=OPTIONS) + + errors = validate_answers(form, {str(question.pk): "carnivore"}) + + assert str(question.pk) in errors + + +def test_multi_select_must_be_a_list(): + form = FormFactory() + question = _question( + FormQuestion.QuestionType.MULTI_SELECT, form=form, options=OPTIONS + ) + + errors = validate_answers(form, {str(question.pk): "vegan"}) + + assert str(question.pk) in errors + + +def test_multi_select_rejects_a_single_unknown_item(): + form = FormFactory() + question = _question( + FormQuestion.QuestionType.MULTI_SELECT, form=form, options=OPTIONS + ) + + errors = validate_answers(form, {str(question.pk): ["vegan", "carnivore"]}) + + assert str(question.pk) in errors + + +def test_boolean_answer_must_be_a_bool(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.BOOLEAN, form=form) + + errors = validate_answers(form, {str(question.pk): "yes"}) + + assert str(question.pk) in errors + + +def test_url_answer_must_be_a_valid_url(): + form = FormFactory() + question = _question(FormQuestion.QuestionType.URL, form=form) + + errors = validate_answers(form, {str(question.pk): "not a url"}) + + assert str(question.pk) in errors + + +def test_multiple_errors_are_collected_per_call(): + form = FormFactory() + required = _question(FormQuestion.QuestionType.TEXT, form=form, required=True) + boolean = _question(FormQuestion.QuestionType.BOOLEAN, form=form) + + errors = validate_answers(form, {str(boolean.pk): "yes"}) + + assert str(required.pk) in errors + assert str(boolean.pk) in errors + + +def test_wrap_and_unwrap_answers_round_trip(): + answers = {"1": "hello", "2": ["a", "b"], "3": False} + + envelope = wrap_answers(answers) + + assert envelope == {"version": 1, "answers": answers} + assert unwrap_answers(envelope) == answers + + +def test_unwrap_answers_rejects_unknown_versions(): + with pytest.raises(ValueError, match="version"): + unwrap_answers({"version": 2, "answers": {}}) From f9c2a273da3012e4a0c962c6f930be2d95f2a930 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:07:38 +0200 Subject: [PATCH 04/13] Add Django admin for authoring forms and browsing answers FormAdmin with inline questions (freeze rule surfaces via model validation so new questions can still be added to answered forms; inline deletion is blocked once answers exist). FormAnswerAdmin is read-only. --- backend/generic_forms/admin.py | 49 +++++++++++++++++++++++ backend/generic_forms/tests/test_admin.py | 45 +++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 backend/generic_forms/admin.py create mode 100644 backend/generic_forms/tests/test_admin.py diff --git a/backend/generic_forms/admin.py b/backend/generic_forms/admin.py new file mode 100644 index 0000000000..4c2e28da42 --- /dev/null +++ b/backend/generic_forms/admin.py @@ -0,0 +1,49 @@ +from django.contrib import admin + +from generic_forms.models import Form, FormAnswer, FormQuestion + + +class FormQuestionInline(admin.TabularInline): + model = FormQuestion + extra = 1 + fields = ( + "label", + "description", + "question_type", + "options", + "required", + "max_length", + "order", + "active", + ) + + # Freezing of question_type/options/required on answered forms is + # enforced by FormQuestion.clean(), which surfaces as a normal form + # error here. Inline-level readonly would also freeze NEW rows, and + # adding questions to an answered form must stay possible. + + def has_delete_permission(self, request, obj=None): + if obj and obj.answers.exists(): + return False + return super().has_delete_permission(request, obj) + + +@admin.register(Form) +class FormAdmin(admin.ModelAdmin): + list_display = ("name", "conference", "purpose") + list_filter = ("conference", "purpose") + search_fields = ("name",) + inlines = [FormQuestionInline] + + +@admin.register(FormAnswer) +class FormAnswerAdmin(admin.ModelAdmin): + list_display = ("form", "user", "created") + list_filter = ("form__conference", "form__purpose") + autocomplete_fields = ("user",) + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False diff --git a/backend/generic_forms/tests/test_admin.py b/backend/generic_forms/tests/test_admin.py new file mode 100644 index 0000000000..16c7c2d305 --- /dev/null +++ b/backend/generic_forms/tests/test_admin.py @@ -0,0 +1,45 @@ +import pytest +from django.contrib.admin.sites import site +from django.test import RequestFactory + +from generic_forms.admin import FormAnswerAdmin, FormQuestionInline +from generic_forms.models import Form, FormAnswer +from generic_forms.tests.factories import FormAnswerFactory, FormFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def admin_request(admin_superuser): + request = RequestFactory().get("/") + request.user = admin_superuser + return request + + +def test_form_and_form_answer_are_registered(): + assert site.is_registered(Form) + assert site.is_registered(FormAnswer) + + +def test_questions_cannot_be_deleted_from_an_answered_form(admin_request): + answer = FormAnswerFactory() + inline = FormQuestionInline(Form, site) + + assert inline.has_delete_permission(admin_request, answer.form) is False + + +def test_questions_can_be_deleted_from_an_unanswered_form(admin_request): + form = FormFactory() + inline = FormQuestionInline(Form, site) + + assert inline.has_delete_permission(admin_request, form) is True + + +def test_form_answers_are_read_only_in_admin(admin_request): + answer_admin = FormAnswerAdmin(FormAnswer, site) + + assert answer_admin.has_add_permission(admin_request) is False + assert answer_admin.has_change_permission(admin_request) is False + assert ( + answer_admin.has_change_permission(admin_request, FormAnswerFactory()) is False + ) From 455525748221dee885ed5f7f28818b480da5e462 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:19:07 +0200 Subject: [PATCH 05/13] Harden generic_forms after adversarial review - validate_answers: reject non-dict answers and non-string multi-select items with errors instead of crashing (client-controlled JSON input) - validate options shape at authoring time (list of unique {id, label} string pairs, required for select types, forbidden otherwise) so a malformed options blob can't 500 every submission - unwrap_answers: ValueError on malformed envelopes, empty dict allowed - block FormAnswer deletion in admin (deleting answers would unfreeze questions and silently destroy submissions) - freeze Form.conference/purpose once answered (model + admin readonly) - question delete guard moved to pre_delete signal so queryset deletes are covered; explicit-pk saves no longer crash (_state.adding) --- backend/generic_forms/admin.py | 11 +- backend/generic_forms/models.py | 94 +++++- backend/generic_forms/services.py | 19 +- backend/generic_forms/tests/test_admin.py | 16 + backend/generic_forms/tests/test_models.py | 64 ++++ backend/generic_forms/tests/test_services.py | 32 ++ specs/generic-form-system.md | 303 ++++++++++++++++++ tasks/generic-forms/plan.md | 316 +++++++++++++++++++ tasks/generic-forms/todo.md | 32 ++ 9 files changed, 876 insertions(+), 11 deletions(-) create mode 100644 specs/generic-form-system.md create mode 100644 tasks/generic-forms/plan.md create mode 100644 tasks/generic-forms/todo.md diff --git a/backend/generic_forms/admin.py b/backend/generic_forms/admin.py index 4c2e28da42..279bd6d4db 100644 --- a/backend/generic_forms/admin.py +++ b/backend/generic_forms/admin.py @@ -35,15 +35,24 @@ class FormAdmin(admin.ModelAdmin): search_fields = ("name",) inlines = [FormQuestionInline] + def get_readonly_fields(self, request, obj=None): + if obj and obj.answers.exists(): + return ("conference", "purpose") + return () + @admin.register(FormAnswer) class FormAnswerAdmin(admin.ModelAdmin): list_display = ("form", "user", "created") list_filter = ("form__conference", "form__purpose") - autocomplete_fields = ("user",) def has_add_permission(self, request): return False def has_change_permission(self, request, obj=None): return False + + def has_delete_permission(self, request, obj=None): + # deleting answers would unfreeze the form's questions and silently + # destroy an applicant's submission + return False diff --git a/backend/generic_forms/models.py b/backend/generic_forms/models.py index f4272bf9b0..1c55a6ab14 100644 --- a/backend/generic_forms/models.py +++ b/backend/generic_forms/models.py @@ -1,5 +1,7 @@ from django.core.exceptions import ValidationError from django.db import models +from django.db.models.signals import pre_delete +from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ from model_utils.models import TimeStampedModel @@ -19,6 +21,38 @@ class Purpose(models.TextChoices): purpose = models.CharField(max_length=32, choices=Purpose.choices) name = models.CharField(max_length=200) + # Frozen once any answer exists: moving a form to another conference or + # purpose would re-contextualize the stored answers. + FROZEN_FIELDS = ("conference_id", "purpose") + + def save(self, *args, **kwargs): + self._check_frozen_fields() + super().save(*args, **kwargs) + + def clean(self): + super().clean() + self._check_frozen_fields() + + def _check_frozen_fields(self): + if self._state.adding: + return + + stored = Form.objects.filter(pk=self.pk).first() + if stored is None: + return + + changed = [ + field + for field in self.FROZEN_FIELDS + if getattr(stored, field) != getattr(self, field) + ] + if changed and stored.answers.exists(): + fields = ", ".join(field.removesuffix("_id") for field in changed) + raise ValidationError( + f"The form already has answers; these fields cannot be " + f"changed: {fields}." + ) + def __str__(self): return f"{self.name} ({self.purpose}, {self.conference.name})" @@ -59,29 +93,63 @@ class QuestionType(models.TextChoices): # Fields that define what an answer means; frozen once any answer exists # so stored answers always match their questions. label/description/order/ # active stay editable (deactivate instead of delete). + # Enforced in save()/clean() and a pre_delete signal: QuerySet.update() + # and bulk_update() bypass both, so they must never be used on this model. + # The exists()-then-write window is not locked; a first answer racing an + # edit is accepted as a non-issue at this scale. FROZEN_FIELDS = ("form_id", "question_type", "options", "required") + CHOICE_TYPES = (QuestionType.SELECT, QuestionType.MULTI_SELECT) def clean(self): super().clean() + self._validate_options() self._check_frozen_fields() def save(self, *args, **kwargs): + self._validate_options() self._check_frozen_fields() super().save(*args, **kwargs) - def delete(self, *args, **kwargs): - if self.form.answers.exists(): + def _validate_options(self): + if self.question_type not in self.CHOICE_TYPES: + if self.options: + raise ValidationError( + {"options": "Only select questions can have options."} + ) + return + + if not isinstance(self.options, list) or not self.options: raise ValidationError( - "This question cannot be deleted because the form already has " - "answers. Deactivate it instead." + {"options": "Select questions need a non-empty list of options."} ) - return super().delete(*args, **kwargs) + + for option in self.options: + if ( + not isinstance(option, dict) + or not isinstance(option.get("id"), str) + or not option["id"] + or not isinstance(option.get("label"), str) + or not option["label"] + ): + raise ValidationError( + { + "options": 'Every option must be {"id": "...", ' + '"label": "..."} with non-empty strings.' + } + ) + + ids = [option["id"] for option in self.options] + if len(ids) != len(set(ids)): + raise ValidationError({"options": "Option ids must be unique."}) def _check_frozen_fields(self): - if not self.pk: + if self._state.adding: + return + + stored = FormQuestion.objects.filter(pk=self.pk).first() + if stored is None: return - stored = FormQuestion.objects.get(pk=self.pk) changed = [ field for field in self.FROZEN_FIELDS @@ -128,3 +196,15 @@ class Meta: name="unique_form_answer_per_user", ) ] + + +# pre_delete (not FormQuestion.delete) so QuerySet.delete() and cascades are +# guarded too. Forms with answers cannot cascade here: FormAnswer.form is +# PROTECT, so only questions of unanswered forms ever reach deletion. +@receiver(pre_delete, sender=FormQuestion) +def block_deleting_answered_questions(sender, instance, **kwargs): + if instance.form.answers.exists(): + raise ValidationError( + "This question cannot be deleted because the form already has " + "answers. Deactivate it instead." + ) diff --git a/backend/generic_forms/services.py b/backend/generic_forms/services.py index a37598514a..317d8e9131 100644 --- a/backend/generic_forms/services.py +++ b/backend/generic_forms/services.py @@ -13,16 +13,27 @@ def wrap_answers(answers: dict) -> dict: def unwrap_answers(envelope: dict) -> dict: + if not isinstance(envelope, dict): + raise ValueError("Malformed answers envelope.") + if envelope == {}: + # a FormAnswer created without going through wrap_answers + return {} version = envelope.get("version") if version != ANSWERS_VERSION: raise ValueError(f"Unknown answers version: {version}") - return envelope["answers"] + answers = envelope.get("answers") + if not isinstance(answers, dict): + raise ValueError("Malformed answers envelope: missing answers map.") + return answers def validate_answers(form: Form, answers: dict) -> dict[str, list[str]]: """Validate a flat {question_id: value} map against the form's active questions. Returns {question_id: [error messages]}; empty dict when valid. """ + if not isinstance(answers, dict): + return {"__all__": ["Invalid answers format."]} + errors: dict[str, list[str]] = defaultdict(list) questions = { str(question.pk): question for question in form.questions.filter(active=True) @@ -69,8 +80,10 @@ def _validate_value(question: FormQuestion, value) -> list[str]: return [] if question_type == types.MULTI_SELECT: - if not isinstance(value, list): - return ["Invalid value: expected a list of options."] + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + return ["Invalid value: expected a list of option ids."] invalid = [item for item in value if item not in _option_ids(question)] if invalid: return ["Invalid options: " + ", ".join(map(str, invalid)) + "."] diff --git a/backend/generic_forms/tests/test_admin.py b/backend/generic_forms/tests/test_admin.py index 16c7c2d305..8f676c4241 100644 --- a/backend/generic_forms/tests/test_admin.py +++ b/backend/generic_forms/tests/test_admin.py @@ -43,3 +43,19 @@ def test_form_answers_are_read_only_in_admin(admin_request): assert ( answer_admin.has_change_permission(admin_request, FormAnswerFactory()) is False ) + assert answer_admin.has_delete_permission(admin_request) is False + assert ( + answer_admin.has_delete_permission(admin_request, FormAnswerFactory()) is False + ) + + +def test_form_conference_and_purpose_become_readonly_once_answered(admin_request): + form_admin = site._registry[Form] + + assert form_admin.get_readonly_fields(admin_request, FormFactory()) == () + + answer = FormAnswerFactory() + assert form_admin.get_readonly_fields(admin_request, answer.form) == ( + "conference", + "purpose", + ) diff --git a/backend/generic_forms/tests/test_models.py b/backend/generic_forms/tests/test_models.py index de3391b195..2235ece659 100644 --- a/backend/generic_forms/tests/test_models.py +++ b/backend/generic_forms/tests/test_models.py @@ -126,3 +126,67 @@ def test_new_question_can_be_added_to_answered_form(): question = _answered_question() FormQuestionFactory(form=question.form) + + +def test_new_question_can_be_saved_with_an_explicit_pk(): + form = FormFactory() + + FormQuestion( + pk=987654, + form=form, + label="Explicit pk", + question_type=FormQuestion.QuestionType.TEXT, + ).save() + + assert FormQuestion.objects.filter(pk=987654).exists() + + +def test_queryset_delete_cannot_remove_questions_from_an_answered_form(): + question = _answered_question() + + with pytest.raises(ValidationError, match="deleted"): + FormQuestion.objects.filter(pk=question.pk).delete() + + +def test_select_options_must_be_a_list_of_id_label_dicts(): + with pytest.raises(ValidationError, match="options"): + FormQuestionFactory( + question_type=FormQuestion.QuestionType.SELECT, + options=["vegan", 42], + ) + + +def test_select_options_cannot_be_empty(): + with pytest.raises(ValidationError, match="options"): + FormQuestionFactory(question_type=FormQuestion.QuestionType.SELECT, options=[]) + + +def test_option_ids_must_be_unique(): + with pytest.raises(ValidationError, match="options"): + FormQuestionFactory( + question_type=FormQuestion.QuestionType.MULTI_SELECT, + options=[{"id": "a", "label": "A"}, {"id": "a", "label": "Again"}], + ) + + +def test_non_choice_questions_cannot_have_options(): + with pytest.raises(ValidationError, match="options"): + FormQuestionFactory( + question_type=FormQuestion.QuestionType.TEXT, + options=[{"id": "a", "label": "A"}], + ) + + +def test_form_conference_and_purpose_are_frozen_once_answered(): + answer = FormAnswerFactory(form__purpose=Form.Purpose.GRANT) + + answer.form.purpose = Form.Purpose.GENERIC + with pytest.raises(ValidationError, match="purpose"): + answer.form.save() + + +def test_form_name_stays_editable_once_answered(): + answer = FormAnswerFactory() + + answer.form.name = "Renamed" + answer.form.save() diff --git a/backend/generic_forms/tests/test_services.py b/backend/generic_forms/tests/test_services.py index 19ec0c82f1..948db85247 100644 --- a/backend/generic_forms/tests/test_services.py +++ b/backend/generic_forms/tests/test_services.py @@ -127,6 +127,25 @@ def test_multi_select_must_be_a_list(): assert str(question.pk) in errors +def test_multi_select_rejects_non_string_items_without_crashing(): + form = FormFactory() + question = _question( + FormQuestion.QuestionType.MULTI_SELECT, form=form, options=OPTIONS + ) + + errors = validate_answers(form, {str(question.pk): [["vegan"]]}) + + assert str(question.pk) in errors + + +def test_non_dict_answers_return_a_global_error(): + form = FormFactory() + + errors = validate_answers(form, ["not", "a", "dict"]) + + assert errors == {"__all__": ["Invalid answers format."]} + + def test_multi_select_rejects_a_single_unknown_item(): form = FormFactory() question = _question( @@ -179,3 +198,16 @@ def test_wrap_and_unwrap_answers_round_trip(): def test_unwrap_answers_rejects_unknown_versions(): with pytest.raises(ValueError, match="version"): unwrap_answers({"version": 2, "answers": {}}) + + +def test_unwrap_answers_treats_empty_envelope_as_no_answers(): + assert unwrap_answers({}) == {} + + +def test_unwrap_answers_rejects_malformed_envelopes(): + with pytest.raises(ValueError): + unwrap_answers(["not", "a", "dict"]) + with pytest.raises(ValueError): + unwrap_answers({"version": 1}) + with pytest.raises(ValueError): + unwrap_answers({"version": 1, "answers": "not a dict"}) diff --git a/specs/generic-form-system.md b/specs/generic-form-system.md new file mode 100644 index 0000000000..da980eba3e --- /dev/null +++ b/specs/generic-form-system.md @@ -0,0 +1,303 @@ +# Spec: Generic Form System + +Status: Approved — ready for planning +Source: Notion draft "Generic Form system" (exported HTML in repo root) + clarifying Q&A +Author: generated via spec-driven-development + +--- + +## 1. Objective + +Build a generic, per-conference configurable form system so organizers can change the questions asked in recurring flows (grants, CFP, visa, feedback) **without backend or frontend code changes**. Today every question is a hardcoded model column (`Grant`, `Submission`) or an external Google Form; changing questions for a new conference edition requires coordinated BE + FE work and migrations. + +**First consumer (this spec's scope): the grant application form.** The engine is built generically; grants is the first flow wired to it. CFP, visa, and feedback forms are explicitly future slices. + +**Target users:** +- *Organizers* — author/edit form questions per conference in Django admin. +- *Attendees/applicants* — fill forms on the Next.js frontend. +- *Maintainers* — stop writing migrations + form components for every question change. + +**Success looks like:** an organizer can add, reword, reorder, or deactivate a grant-form question for the next conference entirely from Django admin, and the frontend renders and validates it with zero code changes. + +### Decisions already made (via Q&A) + +1. **MVP integration target: grants** (biggest pain; `Grant` has ~20 hardcoded answer columns). +2. **Data model: hybrid** — `Form`/`FormQuestion` as normal models (admin-authorable), answers stored as a single `FormAnswer` row per submission with a `JSONField` mapping `question_id → value`. No per-question answer rows. +3. **Versioning: freeze-on-answer** — a question's semantic fields (type, options, required) become immutable once any answer exists for its form. Changes happen by deactivating questions and adding new ones (or cloning the form for a new conference). No snapshot or version-row machinery. +4. **Authoring UI: Django admin** — inline `FormQuestion` editing under `Form`. No custom-admin/Astro builder in this slice. +5. **Load-bearing grant fields stay as `Grant` columns** (confirmed). Fields that drive business logic — `grant_type` (reimbursement categories), `departure_country`/`nationality` (`country_type` derivation, visa), `departure_city`, `needs_funds_for_travel`, `need_visa`, `need_accommodation` — remain structured columns on `Grant`, as do `full_name`/`name`. The *soft* questions moving into the generic form are exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Corrected during planning: socials/website do NOT move — the grant form's social inputs are `participant_*` fields handled via `PublicProfileCard`/`Participant` upsert, not Grant columns; Grant's own social columns are already unused by the current flow.) This avoids a question→field mapping layer in the MVP. +6. **English only** — no multi-lingual labels/options (confirmed). +7. **Options-as-JSON admin UX**: raw JSON widget is acceptable — no custom widget (confirmed). +8. **Grant admin export includes dynamic answers in this slice** (confirmed). The existing `GrantResource` (django-import-export, `grants/admin.py`) exports several soft-question columns today; those move to dynamic-answer columns — one column per question of the conference's grant form (the export is already single-conference via `before_export`). +9. **`purpose` enum values for cfp/visa/feedback are added when those slices land**, not preemptively (confirmed). + +### Assumptions I'm making (correct before approval if wrong) + +1. **No data migration of historical grants.** Old `Grant` columns stay populated and readable for past conferences; new conferences write soft answers to `FormAnswer` only. Legacy columns become nullable/blank-able but are **not dropped** in this slice. +2. **One `FormAnswer` per (form, user).** Matches the existing one-grant-per-user-per-conference constraint. Multi-response generic forms (e.g. anonymous feedback) are future work. +3. **Question labels/descriptions are editable even after answers exist** (typo fixes); only `question_type`, `options`, and `required` freeze. Deletion is blocked once answered — deactivate instead. +4. **New Django app named `generic_forms`** (avoids collision/confusion with `django.forms` and `wagtail.contrib.forms`, which is installed but unused). +5. **Select options live in a `JSONField` on `FormQuestion`** (list of `{id, label}`), not a third model — Django admin can't nest inlines two levels deep, and options-as-JSON keeps authoring on one page. +6. **No file-upload question type in MVP** — it requires extending `files_upload.File.Type`, size limits, and upload permissions. Listed as future work. +7. **No conditional/branching questions in MVP.** + +--- + +## 2. Scope + +### In scope + +- New `generic_forms` Django app: `Form`, `FormQuestion`, `FormAnswer` models + migrations + admin. +- Question types: `text` (single line), `textarea`, `select`, `multi_select`, `boolean`, `url`. +- Server-side answer validation (required, type, option membership, max length, URL format) following the existing `BaseErrorType` pattern. +- GraphQL: query a conference's form by purpose (id, name, ordered active questions with labels/options); mutation to submit/update answers is folded into the existing grant mutations (see §5). +- Grants integration: `sendGrant`/`updateGrant` accept an `answers` input, validate against the conference's grant form, persist a `FormAnswer` linked from `Grant`. +- Frontend: a reusable `DynamicForm` component (styleguide inputs, `react-use-form-state`) rendering questions by type; grant form page renders its soft-question sections dynamically. +- Django admin: grant admin displays the applicant's dynamic answers read-only alongside the structured fields. +- Grant admin export: `GrantResource` gains one column per question of the conference's grant form, populated from the linked `FormAnswer`; legacy soft-question columns stay for historical exports. +- Freeze-on-answer enforcement at the model layer (not just admin). + +### Out of scope (explicitly NOT in this slice) + +- CFP/Submission, visa, and feedback form integrations (engine supports `purpose` values for them, but no product wiring). +- Migrating historical `Grant` answer data into `FormAnswer`; dropping legacy `Grant` columns. +- Custom-admin (Astro) form-builder UI; Wagtail integration. +- File-upload, date, number, or conditional question types. +- Anonymous / multi-response forms. +- Generic "form submitted" confirmation email plumbing (draft's idea — good future win, not now; grants keeps its existing notification path). +- Changes to Pretix, Stripe, or the reimbursement flow. +- Profile-based prefill of dynamic answers (today `ageGroup` prefills from `user.dateBirth` and `gender` from `user.gender`; the generic engine has no per-question semantics, so these prefills are dropped — small accepted UX regression). + +--- + +## 3. Tech stack + +- **Backend:** Django 5.x (existing), PostgreSQL, Strawberry GraphQL. No new Python dependencies expected. +- **Language:** English only — plain `CharField`/`TextField` for labels, descriptions, option labels. No `I18nCharField`/`I18nTextField`. +- **Frontend:** Next.js (existing), TypeScript, Apollo Client with codegen (`pnpm codegen`), `react-use-form-state` (corrected during planning: `react-hook-form` is in package.json but has zero usages in the codebase — every existing form, including the modern invitation-letter form, uses `react-use-form-state`; the new component follows the actual in-repo pattern), `@python-italia/pycon-styleguide` inputs. +- **Admin:** stock Django admin with `TabularInline`/`StackedInline`. + +--- + +## 4. Data model + +```python +# backend/generic_forms/models.py +class Form(TimeStampedModel): + class Purpose(models.TextChoices): + GRANT = "grant", _("Grant") + GENERIC = "generic", _("Generic") # cfp/visa/feedback added in later slices + + conference = models.ForeignKey("conferences.Conference", on_delete=models.CASCADE, + related_name="forms") + purpose = models.CharField(max_length=32, choices=Purpose.choices) + name = models.CharField(max_length=200) + # constraint: at most one form per (conference, purpose) when purpose != GENERIC + + +class FormQuestion(TimeStampedModel): + class QuestionType(models.TextChoices): + TEXT = "text" + TEXTAREA = "textarea" + SELECT = "select" + MULTI_SELECT = "multi_select" + BOOLEAN = "boolean" + URL = "url" + + form = models.ForeignKey(Form, on_delete=models.CASCADE, related_name="questions") + label = models.CharField(max_length=300) + description = models.TextField(blank=True) + question_type = models.CharField(max_length=32, choices=QuestionType.choices) + options = models.JSONField(blank=True, default=list) + # options item shape: {"id": "vegan", "label": "Vegan"} + required = models.BooleanField(default=False) + max_length = models.PositiveIntegerField(null=True, blank=True) + order = models.PositiveIntegerField(default=0) + active = models.BooleanField(default=True) # deactivate instead of delete once answered + + +class FormAnswer(TimeStampedModel): + form = models.ForeignKey(Form, on_delete=models.PROTECT, related_name="answers") + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + answers = models.JSONField(default=dict) + # Versioned envelope so the structure can evolve without guessing: + # {"version": 1, "answers": {"": value}} + # version 1 value types by question_type: + # text/textarea/url → str, select → option id (str), + # multi_select → list[str] of option ids, boolean → bool + # Readers dispatch on "version"; writers always write the current version. + # (GraphQL input stays the flat {question_id: value} map — the envelope is + # a storage concern; the mutation wraps it on persist.) + + class Meta: + constraints = [models.UniqueConstraint(fields=["form", "user"], + name="unique_form_answer_per_user")] +``` + +**Grant link:** `Grant.form_answer = models.OneToOneField("generic_forms.FormAnswer", null=True, blank=True, on_delete=models.SET_NULL)`. Soft-question columns on `Grant` become `blank=True` (kept for historical data). + +**Freeze-on-answer rule (model layer):** `FormQuestion.clean()`/`save()` raise if `question_type`, `options`, or `required` change while `self.form.answers.exists()`; deletion is blocked via a `pre_delete` signal (covers queryset deletes too). `label`/`description`/`order`/`active` stay editable. `Form.conference`/`purpose` freeze the same way. In admin the rule surfaces as validation errors on the inline (not readonly fields — inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); the model is the enforcement point. Question `options` are shape-validated at authoring (list of `{id, label}` string pairs, unique ids, required for select types, forbidden otherwise). + +**Answer validation (single source of truth):** a `validate_answers(form, answers: dict) -> dict[str, list[str]]` service in `generic_forms/` used by the GraphQL layer: unknown/inactive question ids rejected, required enforced, per-type checks (option membership incl. every item of multi_select, `URLValidator` for url, `max_length` for text types, bool type check). + +--- + +## 5. API design (GraphQL) + +Follows the newer one-mutation-per-file pattern and the `api/visa/mutations/request_invitation_letter.py` validation style. + +**Query** — extend the existing `Conference` type: + +```graphql +conference(code: "pycon2026") { + form(purpose: GRANT) { # null if no form configured + id + name + questions { # active only, ordered + id + label + description + questionType + required + maxLength + options { id label } + } + } +} +``` + +**Mutations** — no standalone `submitFormAnswers` in this slice. `sendGrant` / `updateGrant` inputs gain an optional `answers: JSON` (map of question id → value). The mutation: +1. Keeps its existing deadline gating unchanged (`non_field_errors: "The grants form is not open!"` via `Conference.is_grants_open`) — no `FormNotAvailable` union member (changing the deadline-closed response shape would break the deployed frontend; decided during planning). +2. If `answers` is provided but the conference has no `GRANT` form, rejects with a clear error. If the form exists, runs `validate_answers`; failures are returned in a dedicated `answersErrors: JSON` field on `GrantErrors` mapping `question_id → [messages]`. (Dotted dynamic paths like `answers.` cannot serialize through the statically-typed error classes — verified during planning; the in-repo dotted-path precedent, `materials.0.url` in `api/submissions`, works only because its container field is statically declared.) +3. Persists `FormAnswer` (create or update), wrapping the input map into the versioned envelope (`{"version": 1, "answers": {...}}`), and links it to the `Grant` in the same transaction. +4. The 8 legacy soft input fields become optional; legacy-shape submissions (soft fields, no `answers`) keep working unchanged until the frontend cutover, then get removed in a post-deploy follow-up. + +**Grant type** — exposes `formAnswers: JSON | null` (the unwrapped flat map) so the frontend edit flow can prefill the dynamic questions. + +Privacy policy acceptance, Slack notification, and email template lookups keep their current grant-specific wiring — unchanged. + +--- + +## 6. Commands + +All backend commands run inside Docker (per CLAUDE.md). + +| Purpose | Command | +|---|---| +| Run backend tests (new app) | `docker exec pycon-backend-1 uv run pytest generic_forms/tests api/generic_forms -l -s -vvv` | +| Grants integration tests | `docker exec pycon-backend-1 uv run pytest api/grants grants -l -s -vvv` | +| Full suite | `docker exec pycon-backend-1 uv run pytest` | +| Make migrations | `docker exec pycon-backend-1 uv run python manage.py makemigrations generic_forms grants` | +| Migrate | `docker exec pycon-backend-1 uv run python manage.py migrate` | +| Lint / format | `docker exec pycon-backend-1 uv run ruff check` / `uv run ruff format` | +| Type check | `docker exec pycon-backend-1 uv run mypy .` | +| Frontend codegen (after schema change) | `cd frontend && pnpm codegen` | +| Frontend tests / build | `cd frontend && pnpm test` / `pnpm build` | + +--- + +## 7. Project structure + +``` +backend/ + generic_forms/ # NEW app + models.py # Form, FormQuestion, FormAnswer + services.py # validate_answers() + admin.py # Form admin + FormQuestion inline (freeze-aware) + migrations/ + tests/ # model + validation tests, factories + api/ + generic_forms/ # NEW: FormType, FormQuestionType (query side) + types.py + grants/mutations.py # extend sendGrant/updateGrant with answers + grants/ + models.py # + form_answer FK; soft columns → blank=True + admin.py # + read-only answers display + pycon/settings/base.py # + generic_forms in INSTALLED_APPS + +frontend/src/ + components/dynamic-form/ # NEW: renders FormQuestion[] via styleguide inputs + index.tsx + form.graphql # fragment for form + questions + components/grant-form/ # integrate DynamicForm for soft questions +``` + +--- + +## 8. Code style + +Backend follows existing conventions — Ruff (lint + format), mypy clean. Mutation validation mirrors the in-repo pattern: + +```python +@strawberry.input +class SendGrantInput: + conference: strawberry.ID + answers: JSON + ... + + def validate(self, conference: Conference, form: Form) -> GrantErrors | None: + errors = GrantErrors() + if answer_errors := validate_answers(form, self.answers): + # dedicated JSON field: {question_id: [messages]} — dynamic keys + # cannot serialize through the statically-typed error fields + errors.answers_errors = answer_errors + return errors.if_has_errors +``` + +Frontend: `react-use-form-state` + `@python-italia/pycon-styleguide` primitives (mirror `invitation-letter-form.tsx`: `InputWrapper` around each field, `MultiplePartsCard` sections); GraphQL documents co-located with components; **never hand-edit generated files** (`src/types.tsx`, `src/generated/`). + +--- + +## 9. Testing strategy + +- **Framework:** pytest + factory-based fixtures, in-app `tests/` dirs (existing convention). Frontend: existing `pnpm test` setup for the `DynamicForm` component's rendering/validation mapping. +- **Model tests** (`generic_forms/tests/`): freeze-on-answer (type/options/required change and delete blocked once an answer exists; label/order/active edits allowed); unique (form, user) constraint; one-form-per-(conference, purpose) constraint. +- **Validation tests:** each question type's accept/reject cases — required missing, wrong value type, unknown question id, inactive question id, non-member option, multi_select with one bad item, invalid URL, over max_length. +- **API tests** (`api/` tests): query returns only active questions in order; `sendGrant` with valid answers creates `Grant` + linked `FormAnswer` atomically; invalid answers return per-question errors in `answersErrors` and persist nothing; answers-with-no-form-configured is rejected; grants-deadline-closed behavior unchanged from today; an answers-only payload omitting all 8 legacy soft fields succeeds end-to-end (this is the exact post-cutover frontend payload). +- **Export test:** `GrantResource` export of a grant with a linked `FormAnswer` produces one column per form question with the answer values (option ids resolved to labels); grants without `FormAnswer` (historical) still export cleanly. +- **Regression:** full existing grants test suite stays green — legacy columns still accepted for old data paths. +- Every slice lands with its tests; `pytest`, `ruff check`, `mypy .` green before any commit. + +--- + +## 10. Boundaries + +### Always do +- Run backend commands via `docker exec pycon-backend-1 ...` (local venv doesn't work). +- Run `pytest` + `ruff check` + `mypy .` (and `pnpm codegen` after schema changes) before committing. +- Enforce freeze-on-answer in the model, not only in admin. +- Validate answers server-side via `validate_answers` — frontend validation is UX only. +- Keep legacy `Grant` columns readable (admin, exports) for historical conferences. + +### Ask first +- Adding any new dependency (backend or frontend). +- Changing which `Grant` fields count as load-bearing (decision #5) — i.e. moving `grant_type`, country, or `need_*` fields into the form. +- Any data migration touching existing `Grant` rows beyond `blank=True` loosening. +- Adding new values to `files_upload.File.Type` (file-upload question type). +- Schema changes to `Submission`, visa, or notification models. +- Dropping or renaming any existing column. + +### Never +- Drop legacy `Grant` answer columns in this slice. +- Hand-edit generated GraphQL types (`frontend/src/types.tsx`, `*.generated.ts`). +- Store answers as per-question rows (decision: JSON) or bypass `validate_answers` in any mutation. +- Commit secrets; weaken rate-limit/permission classes on mutations. +- Delete or skip failing tests to get green. + +--- + +## 11. Success criteria + +1. Organizer creates a `GRANT` form with questions of every supported type in Django admin, reorders and deactivates questions — no code change needed. +2. Once one answer exists, changing a question's type/options/required or deleting it fails with a clear error in both admin and direct model save; label typo fix still succeeds. +3. `conference.form(purpose: GRANT)` returns the ordered active questions; returns `null` when unconfigured. +4. `sendGrant` with valid `answers` creates `Grant` + linked `FormAnswer` in one transaction; a second submit by the same user for the same conference updates rather than duplicates (existing update path). +5. `sendGrant` with an invalid answer (missing required, bad option, invalid URL) returns per-question errors (`answersErrors` map) and writes nothing; an answers-only payload with no legacy soft fields succeeds. +6. Grant form page on the frontend renders the soft-question sections from the API (verify: add a question in admin → it appears on the page after reload, no deploy of new code). +7. Grant admin shows the applicant's dynamic answers read-only next to structured fields. +8. Grant admin export includes a column per form question with the applicant's answers; exports of historical grants (no `FormAnswer`) still work. +9. Full backend test suite, `ruff check`, `mypy .`, frontend `pnpm build` + `pnpm test` all green. + +## 12. Open questions + +None — all resolved into decisions #7–#9. diff --git a/tasks/generic-forms/plan.md b/tasks/generic-forms/plan.md new file mode 100644 index 0000000000..add0215ed1 --- /dev/null +++ b/tasks/generic-forms/plan.md @@ -0,0 +1,316 @@ +# Implementation Plan: Generic Form System + +Source spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Mode: plan (read-only, no code changed) +Structure: **5 stacked PRs** — each PR is independently mergeable and deployable, stacked in order. +Reviewed: adversarial verify pass (3 independent critics) applied — see "Verified constraints" below. + +## Overview + +Build the `generic_forms` engine (Form / FormQuestion / FormAnswer, freeze-on-answer, JSON answers with versioned envelope), expose it over GraphQL, wire grants as the first consumer (8 soft questions move from hardcoded `Grant` columns to dynamic form answers), surface answers in grant admin + export, and render the form dynamically on the frontend. + +## Resolved since spec (verified in codebase) + +- `react-hook-form` has **zero** usages despite being in package.json; every form (incl. the modern `invitation-letter-form.tsx`) uses `react-use-form-state`. New `DynamicForm` uses `react-use-form-state`. (Spec §3/§8 corrected.) +- Grant's social columns (`website`, `twitter_handle`, …) are **already dead** — not in the GraphQL `Grant` type, not written by the form (socials go through `Participant` via `PublicProfileCard`). They do NOT become form questions. Soft-question set is exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Spec decision #5 corrected.) +- `send_grant`/`update_grant` are `@transaction.atomic` ([api/grants/mutations.py:226,297](../../backend/api/grants/mutations.py)) — FormAnswer persistence slots into the existing transaction. +- `BaseGrantInput.validate()` (mutations.py:74-111) **mixes** structured-field checks (`full_name`, `grant_type`, departure fields — these STAY) with soft-field checks (max lengths why:1000, python_usage:700, been_to_other_events:500, community_contribution:900, notes:350; required: why, python_usage, been_to_other_events). Only the soft-field portion is superseded by `validate_answers` — structured-field validation must remain untouched. +- Of the 8 soft columns, exactly **4** lack `blank=True` today: `why`, `python_usage`, `been_to_other_events`, `occupation`. The other 4 (`age_group`, `community_contribution`, `gender`, `notes`) are already `blank=True`. All 8 are NOT NULL at the DB level (`blank=True` is Python-only) — `None` must never reach `Grant.objects.create`. +- django-import-export is **3.3.9**; dynamic per-export fields are supported: `Resource.__init__` deep-copies `self.fields` (sanctioned mutation point), and `GrantAdmin.get_export_resource_kwargs(request, ...)` passes context into `GrantResource.__init__`. Extra instance fields auto-append to export order. +- Conference GraphQL pattern to mirror: `deadline(self, info, type: str)` at [api/conferences/types.py:196](../../backend/api/conferences/types.py#L196). Enum pattern: `strawberry.enum(Model.TextChoices)`. +- Tests: model tests in `generic_forms/tests/`, API tests in `api/generic_forms/tests/` + `api/grants/tests/`; `graphql_client` fixture, factory_boy, `pytest.mark.django_db`. +- No read-only-JSON admin precedent exists — the answers display in GrantAdmin is net-new (simple `format_html` list, no new deps). + +## Verified constraints (from the adversarial review — these shape the tasks) + +1. **Dotted `answers.` error paths are impossible.** `BaseErrorType.add_error` getattr-traverses statically-typed error classes (api/types.py:33-74); dynamic keys raise `AttributeError`, and strawberry cannot serialize dynamic field names regardless. The in-repo dotted precedent (`materials.0.url`) lives in **api/submissions** (not visa) and works only because `materials: list[ProposalMaterialErrors]` is statically declared. **Decision (resolved, not a risk): `answers_errors: JSON` field on `_GrantErrors`, set by direct assignment.** Spec §5/§8/§11 updated. Frontend consumes `answersErrors` only. +2. **PR3 must survive the exact PR5 payload.** An answers-only submission (all 8 soft fields omitted) must pass: (a) legacy soft-field required/max-length checks run ONLY on the legacy path (answers not provided); (b) soft input `None` values coalesce to `""` before `Grant.objects.create` / the update setattr loop (DB columns are NOT NULL). A named PR3 test sends answers and omits all 8 soft fields. +3. **Frontend codegen needs a deployed backend schema.** `codegen.yml` fetches the schema from a live endpoint; PR CI (`frontend-lint.yml`) codegens against the staging backend (pastaporto), which deploys only via manual `workflow_dispatch`. **PR5 therefore build-depends on PR3 being deployed to staging**, not merely merged. Release step added before PR5. (Optional improvement, needs approval per spec boundaries — CI change: check in a schema snapshot via `strawberry export-schema` and point codegen at the file.) +4. **Deadline-closed behavior stays as-is** (`non_field_errors: "The grants form is not open!"`). No `FormNotAvailable` union member — changing the response shape breaks the deployed frontend. Spec §5 amended accordingly. `answers` with no GRANT form configured → clear field error. +5. **Production data dependency:** the GRANT form must exist (with the 8 questions) in production admin BEFORE PR5 deploys, or the live form loses its soft questions. Seeding command was explicitly cut from scope → this is a manual ops step in Checkpoint 5, on both staging and production. Frontend must also handle `form == null` by blocking submission with a "form not available" state (never submit without answers). +6. **Legacy-field removal follow-up must be two PRs**, not one: (1) frontend-only — strip legacy `GrantErrors` validation selections (submit-grant.graphql:12-34, pages/grants/edit/update-grant.graphql:25-52) and legacy soft-field selections (my-grant.graphql, update-grant.graphql) — deployable against the unchanged backend; (2) after deploy + soak (stale browser tabs still send old payloads), backend-only — remove the legacy input fields. PR5 already stops *sending* soft fields; it also strips whatever legacy selections it can without breaking its own build. + +## Architecture decisions + +- **Stacked-PR back-compat rule:** every PR leaves `main` deployable (backend deploys before frontend, per deploy.yml ordering). PR3 is strictly additive on the wire: soft fields optional, `answers` optional, legacy shape untouched. +- **Answers storage:** versioned envelope `{"version": 1, "answers": {"": value}}`; GraphQL wire format is the flat map (`strawberry.scalars.JSON`). +- **Question ids as answer keys:** `FormQuestion.pk` stringified; frontend treats them as opaque. +- **Prefill regression accepted and specced** (spec §2 out-of-scope): dateBirth→ageGroup and user.gender prefills drop. +- **Mid-cycle cutover caveat:** grants submitted pre-PR5 (legacy path) have soft answers in columns, not FormAnswer — post-cutover their edit view shows empty dynamic questions. Mitigation: deploy the cutover before grants open for the next conference (ops note in Checkpoint 5); a data backfill is explicitly out of scope. + +## Dependency graph + +``` +PR1 generic_forms app (models + freeze + validate_answers + admin) + ├── PR2 GraphQL query side (Conference.form(purpose)) + └── PR3 grants backend (Grant.form_answer, mutations, Grant.formAnswers) + ├── PR4 grant admin display + export (needs PR3 merged) + └── PR5 frontend DynamicForm + grant form (needs PR2 + PR3 DEPLOYED to staging for codegen/CI) +``` + +Linear stack order: PR1 → PR2 → PR3 → PR4 → PR5. PR4 can start once PR3 merges; PR5 once PR3 reaches staging. + +--- + +## PR1 — `generic_forms` app core (backend only, no consumers) + +Suggested branch: `generic-forms/01-app` + +### Task 1.1: App skeleton + models + migration + +**Description:** Create the `generic_forms` Django app with `Form`, `FormQuestion`, `FormAnswer` models per spec §4 (plain `CharField`/`TextField`, English only), DB constraints, and initial migration. Register in `INSTALLED_APPS` (dotted AppConfig path, `default_auto_field = BigAutoField` like `visa/apps.py`). + +**Acceptance criteria:** +- [ ] Models match spec §4: `Form(conference, purpose, name)`, `FormQuestion(form, label, description, question_type, options, required, max_length, order, active)`, `FormAnswer(form PROTECT, user, answers JSON)`. +- [ ] Constraints enforced at DB level: unique `(form, user)` on FormAnswer; at most one form per `(conference, purpose)` when purpose != `generic` (conditional UniqueConstraint). +- [ ] Migration is plain `makemigrations` output; applies cleanly. + +**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; `uv run python manage.py makemigrations --check --dry-run` clean afterward. + +**Dependencies:** None. +**Files:** `backend/generic_forms/{__init__,apps,models}.py`, `backend/generic_forms/migrations/0001_initial.py`, `backend/pycon/settings/base.py`, `backend/generic_forms/tests/{__init__,factories,test_models}.py` +**Scope:** M + +### Task 1.2: Freeze-on-answer enforcement + +**Description:** Once `form.answers.exists()`: changing `question_type`/`options`/`required` on a `FormQuestion`, or deleting it, raises `ValidationError`; `label`/`description`/`order`/`active` stay editable. Enforced in the model (`clean()` + `save()` guard + `delete()` override). + +**Acceptance criteria:** +- [ ] Semantic-field change on an answered form raises; same change on an unanswered form succeeds. +- [ ] Delete blocked on answered form; `active=False` allowed. +- [ ] Label/description/order edits always allowed. + +**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_models.py -l` green. + +**Dependencies:** 1.1. +**Files:** `backend/generic_forms/models.py`, `backend/generic_forms/tests/test_models.py` +**Scope:** S + +### Task 1.3: `validate_answers` service + envelope helpers + +**Description:** `validate_answers(form, answers: dict) -> dict[str, list[str]]` per spec §4 (unknown/inactive ids, required, per-type checks, option membership incl. every multi_select item, `URLValidator`, `max_length`), plus `wrap_answers` / `unwrap_answers` envelope helpers dispatching on `version`. + +**Acceptance criteria:** +- [ ] Every question type has accept + reject cases covered by tests (spec §9 list). +- [ ] Valid input returns `{}`; errors keyed by question id (this dict is exactly what `answers_errors` carries on the wire in PR3). +- [ ] Envelope round-trip: `unwrap(wrap(x)) == x`; unwrap raises on unknown version. + +**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_services.py -l` green. + +**Dependencies:** 1.1. +**Files:** `backend/generic_forms/services.py`, `backend/generic_forms/tests/test_services.py` +**Scope:** M + +### Task 1.4: Django admin for form authoring + +**Description:** `FormAdmin` with `FormQuestionInline` (TabularInline, mirror `SponsorLevelBenefitInline` simplicity; ordered by `order`), raw JSON widget for `options` (decision #7). Freeze rule surfaces as model validation errors in the inline (deviation applied during build: inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); inline deletion blocked once answered; `Form.conference`/`purpose` readonly once answered. `FormAnswerAdmin` fully read-only (no add/change/delete — deleting answers would unfreeze questions and destroy submissions). + +**Acceptance criteria:** +- [ ] Organizer can create a form + questions of every type entirely in admin (success criterion 1). +- [ ] Inline shows semantic fields readonly once the form has answers. +- [ ] FormAnswer visible but not editable in admin. + +**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; manual: create form with all 6 question types in local admin. + +**Dependencies:** 1.2. +**Files:** `backend/generic_forms/admin.py`, `backend/generic_forms/tests/test_admin.py` +**Scope:** S + +### ▣ CHECKPOINT 1 (end of PR1) +- [ ] `pytest generic_forms`, full `pytest`, `ruff check`, `ruff format --check`, `mypy .` all green. +- [ ] PR1 opened; human review before stacking further. + +--- + +## PR2 — GraphQL query side + +Suggested branch: `generic-forms/02-graphql-query` (stacked on PR1) + +### Task 2.1: Form types + `Conference.form(purpose)` field + +**Description:** `api/generic_forms/types.py`: `FormType`, `FormQuestionType` (id, label, description, questionType, required, maxLength, options as `list[FormQuestionOption(id, label)]`), `FormPurpose = strawberry.enum(Form.Purpose)`, `QuestionType = strawberry.enum(FormQuestion.QuestionType)`. Add `form(self, info, purpose: FormPurpose) -> FormType | None` to the Conference type, mirroring `deadline()`. Questions resolver returns active-only, ordered by `order`. + +**Acceptance criteria:** +- [ ] Query in spec §5 works verbatim. +- [ ] Returns `null` when no form configured; inactive questions excluded; order respected. + +**Verification:** `docker exec pycon-backend-1 uv run pytest api/generic_forms -l` green; `ruff`/`mypy` clean. + +**Dependencies:** PR1. +**Files:** `backend/api/generic_forms/{__init__,types}.py`, `backend/api/conferences/types.py`, `backend/api/generic_forms/tests/{__init__,test_form_query}.py` +**Scope:** S + +### ▣ CHECKPOINT 2 (end of PR2) +- [ ] Full backend suite + lint + types green. GraphQL schema diff reviewed (additive only). PR2 opened. + +--- + +## PR3 — grants backend integration + +Suggested branch: `generic-forms/03-grants-backend` (stacked on PR2) + +### Task 3.1: `Grant.form_answer` link + soft-column loosening + +**Description:** Add `Grant.form_answer = OneToOneField(generic_forms.FormAnswer, null=True, blank=True, SET_NULL)`. Loosen the **4** currently-required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`) to `blank=True` (the other 4 already are). One migration, no data changes. Note: columns remain NOT NULL — the mutation layer must never pass `None` (handled in 3.2). + +**Acceptance criteria:** +- [ ] Migration applies; no other schema changes; historical rows untouched. +- [ ] Existing grants test suite green. + +**Verification:** `docker exec pycon-backend-1 uv run pytest grants api/grants -l` green. + +**Dependencies:** PR1. +**Files:** `backend/grants/models.py`, `backend/grants/migrations/00XX_*.py` +**Scope:** S + +### Task 3.2: Mutations accept `answers` (with tests, TDD) + +**Description:** `SendGrantInput`/`UpdateGrantInput`: the 8 soft fields become optional; new optional `answers: JSON`. Validation split: +- Legacy soft-field checks (required + max-length subset of `BaseGrantInput.validate`) run **only** when the legacy path is used (`answers` not provided). Structured-field validation (`full_name`, `grant_type`, departure fields, deadline gating) is **unchanged on both paths**. +- Answers path: reject if no GRANT form configured; else `validate_answers`; failures go into new `answers_errors: JSON` field on `_GrantErrors` by direct assignment (NOT `add_error` — dynamic keys can't traverse the typed class; see Verified constraint 1). +Mutation body: inside the existing `@transaction.atomic`, wrap answers into the envelope, `update_or_create` the FormAnswer, link `grant.form_answer`. Soft input `None` values coalesce to `""` before `Grant.objects.create`; `update_grant`'s `asdict(input)` setattr loop skips `answers` and never writes `None` into soft columns. Tests land in this task (failing-first): answers happy path, invalid answers → `answersErrors` + atomic rollback (no Grant, no FormAnswer), **answers-only payload omitting all 8 soft fields end-to-end (the exact PR5 payload)**, legacy-shape regression (today's payload byte-identical behavior), update-no-duplicate (unique constraint), answers-with-no-form rejected, deadline-closed unchanged, structured-field validation unchanged. + +**Acceptance criteria:** +- [ ] All paths above covered by tests in `api/grants/tests/`; whole grants suite green. +- [ ] Answers-only payload (no soft fields) succeeds — named test. +- [ ] Legacy payload behavior unchanged — named test. +- [ ] No `None` ever written to a NOT NULL soft column (create or update path). + +**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants grants generic_forms -l` green. + +**Dependencies:** 3.1. +**Files:** `backend/api/grants/mutations.py`, `backend/api/grants/tests/test_send_grant.py`, `backend/api/grants/tests/test_update_grant.py` +**Scope:** M + +### Task 3.3: Expose `Grant.formAnswers` (read side) + +**Description:** `formAnswers: JSON | None` on the `Grant` GraphQL type (api/grants/types.py) returning the unwrapped flat map from the linked FormAnswer, `None` when absent. Used by the edit-flow prefill in PR5. Own query test (via `me.grant`). + +**Acceptance criteria:** +- [ ] `me.grant.formAnswers` returns the flat map for a grant with FormAnswer; `null` for a legacy grant. + +**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants -l` green. + +**Dependencies:** 3.2. +**Files:** `backend/api/grants/types.py`, `backend/api/grants/tests/test_grant_type.py` (or existing query test file) +**Scope:** XS + +### ▣ CHECKPOINT 3 (end of PR3) +- [ ] Full suite + lint + mypy green. Schema diff additive. +- [ ] Back-compat verified: legacy payload tests green (old frontend deployable against this backend); answers-only payload test green (new frontend's contract already proven). +- [ ] PR3 opened — **key review gate: back-compat story**. +- [ ] After merge: **deploy to staging (pastaporto) via `workflow_dispatch`** — PR5's CI codegen needs this schema live. + +--- + +## PR4 — grant admin display + export + +Suggested branch: `generic-forms/04-admin` (stacked on PR3; can start once PR3 merges) + +### Task 4.1: Read-only answers display in GrantAdmin + +**Description:** New readonly pseudo-field on `GrantAdmin` (in "The Grant" fieldset) rendering the linked FormAnswer as a question-label → answer list via `format_html` (option ids resolved to labels; multi_select joined). Empty state for historical grants. `select_related`/prefetch on the admin queryset (no N+1). + +**Acceptance criteria:** +- [ ] Grant with FormAnswer shows Q/A pairs readonly; grant without shows an empty note; changelist/change view query counts stay flat. + +**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green; manual admin check. + +**Dependencies:** PR3. +**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` +**Scope:** S + +### Task 4.2: Dynamic export columns + +**Description:** `GrantResource.__init__` accepts export context via `GrantAdmin.get_export_resource_kwargs` (import-export 3.3.9 sanctioned path, verified incl. the export-form preview instantiating with the same kwargs), resolves the conference's GRANT form, appends one `Field` per question (column name = question label, `dehydrate_method` reading the FormAnswer). Historical grants export empty cells; legacy soft columns stay in `EXPORT_GRANTS_FIELDS`. + +**Acceptance criteria:** +- [ ] Export of grants with FormAnswers yields one column per question, values resolved (labels for options). +- [ ] Export of a historical conference (no form/answers) unchanged vs today. + +**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green (resource-level tests). + +**Dependencies:** 4.1 (same files). +**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` +**Scope:** M + +### ▣ CHECKPOINT 4 (end of PR4) +- [ ] Full suite + lint + mypy green. Manual: export CSV from local admin with a seeded form. PR4 opened. + +--- + +## PR5 — frontend DynamicForm + grant form integration + +Suggested branch: `generic-forms/05-frontend` (stacked on PR3; **prerequisite: PR3 deployed to staging** so `frontend-lint` codegen sees the new schema) + +### Task 5.1: `DynamicForm` component + fragment + +**Description:** `frontend/src/components/dynamic-form/`: `form.graphql` fragment (form + questions incl. options), `pnpm codegen`, and `index.tsx` rendering each question by `questionType` via styleguide primitives inside `InputWrapper` (mirror `invitation-letter-form.tsx`): text→`Input`, textarea→`Textarea` (+maxLength), select→`Select`, multi_select→`Checkbox` group, boolean→`Checkbox`, url→`Input`. State via the parent's `react-use-form-state` (answers keyed by question id); errors prop consumes the `answersErrors` map (`question_id → string[]`). + +**Acceptance criteria:** +- [ ] Renders all 6 question types from a fragment-typed prop; required marking + maxLength client-side; per-question errors render under fields. +- [ ] No hand edits to generated files. + +**Verification:** `cd frontend && pnpm codegen && pnpm test && pnpm build` green (component test for render-by-type). + +**Dependencies:** PR2 + PR3 deployed to staging (codegen). +**Files:** `frontend/src/components/dynamic-form/{index.tsx,form.graphql,dynamic-form.test.tsx}` (+ regenerated `src/types.tsx`) +**Scope:** M + +### Task 5.2: Grant form integration — new submission flow + +**Description:** `grant-form/index.tsx`: fetch `conference.form(purpose: GRANT)`; replace the 8 hardcoded soft inputs with `DynamicForm`; build the `answers` map on submit and **stop sending the 8 legacy input fields**; map `answersErrors` to the component. **Null-form guard:** if `form` is `null`, block submission and show a "form not available" state — never submit without answers (Verified constraint 5). Strip the legacy `GrantErrors` validation selections for the 8 soft fields from `submit-grant.graphql`. Structured fields (fullName, nationality, grantType, travel/visa/accommodation, PublicProfileCard, privacy checkbox) untouched. Prune dead `options.ts` constants (`GENDER_OPTIONS`, `AGE_GROUPS_OPTIONS`, `OCCUPATION_OPTIONS`) only if nothing else imports them; `GRANT_TYPE_OPTIONS` stays. Accepted regression (specced): dateBirth/gender prefills drop. + +**Acceptance criteria:** +- [ ] New submission works E2E against local backend with a seeded form (success criterion 6: add question in admin → appears on page, no code change). +- [ ] `form == null` → submission blocked with visible message. +- [ ] Per-question server errors display under the right inputs; no legacy soft fields in the mutation payload. + +**Verification:** `cd frontend && pnpm test && pnpm build`; manual: docker-compose, create form in admin, submit a grant. + +**Dependencies:** 5.1. +**Files:** `frontend/src/components/grant-form/index.tsx`, `frontend/src/components/grant-form/submit-grant.graphql`, `frontend/src/components/grant-form/options.ts` +**Scope:** M + +### Task 5.3: Grant form integration — edit flow + +**Description:** Edit-flow prefill from `me.grant.formAnswers`: add `formAnswers` to `pages/grants/edit/my-grant.graphql`, feed into `DynamicForm` initial state; update `pages/grants/edit/update-grant.graphql` (strip legacy soft-field + validation selections, keep structured ones); `pages/grants/edit/index.tsx` passes the form + answers through. Legacy grants (`formAnswers == null`) show empty dynamic questions — accepted mid-cycle caveat (plan decision; cutover deploys before grants open). + +**Acceptance criteria:** +- [ ] Edit flow prefills dynamic answers and saves changes (update path, no duplicate FormAnswer). +- [ ] `pnpm build` green; edit page documents carry no legacy soft-field selections. + +**Verification:** `cd frontend && pnpm test && pnpm build`; manual: edit a grant submitted via the new flow. + +**Dependencies:** 5.2. +**Files:** `frontend/src/pages/grants/edit/{index.tsx,my-grant.graphql,update-grant.graphql}` +**Scope:** S + +### ▣ CHECKPOINT 5 — FINAL +- [ ] All spec §11 success criteria pass (walk the list one by one). +- [ ] Full backend suite, `ruff`, `mypy`, `pnpm test`, `pnpm build` green. +- [ ] Manual E2E on docker-compose: author form → submit grant → edit grant → view in admin → export CSV. +- [ ] **Ops before merging PR5:** GRANT form with the 8 current questions created and verified in **staging AND production** admin (manual — seeding command was cut from scope). Cutover timed **before grants open** for the next conference (pre-existing legacy applications would show empty dynamic questions in edit). +- [ ] Follow-up ticketed as **two** PRs (not in stack): (1) frontend-only — remove remaining legacy `GrantErrors`/`Grant` selections; (2) after deploy + soak, backend-only — remove legacy soft input fields. + +--- + +## Risks and mitigations + +| Risk | Impact | Mitigation | +|---|---|---| +| PR3 rejects/500s on the future PR5 payload | High | Verified constraint 2 baked into T3.2: conditional legacy validation, None→"" coalescing, named answers-only test | +| PR5 CI codegen can't see PR3 schema | Med | Explicit staging deploy step in Checkpoint 3; optional schema-snapshot improvement (needs approval — CI change) | +| Production GRANT form missing at PR5 deploy → silent soft-answer loss | High | Null-form guard blocks submission (T5.2); manual ops step in Checkpoint 5 for staging + production | +| Mid-cycle cutover: legacy grants' edit view shows empty questions | Med | Deploy before grants open (Checkpoint 5 ops note); backfill explicitly out of scope | +| Legacy-field removal breaks live clients | Med | Follow-up split into frontend-first + soak + backend PRs (Verified constraint 6) | +| Export preview instantiates resource with same kwargs | Low | Known from source read; resource tests cover it | +| `useFormState` dynamic keys awkward for answers record | Low | Single `answers` object in state; component test proves it before integration | + +## Parallelization + +- PR1 tasks sequential (same files). PR2 once PR1 models stable. +- After PR3 **merges**: PR4 can start. After PR3 **reaches staging**: PR5 can start. PR4 ∥ PR5. + +## Open questions + +None. All decisions resolved (error wire format committed: `answersErrors`; deadline behavior unchanged; ops steps explicit). diff --git a/tasks/generic-forms/todo.md b/tasks/generic-forms/todo.md new file mode 100644 index 0000000000..5f09ebaa90 --- /dev/null +++ b/tasks/generic-forms/todo.md @@ -0,0 +1,32 @@ +# TODO: Generic Form System + +Spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Plan: [plan.md](plan.md) +Stack: PR1 → PR2 → PR3 → (PR4 ∥ PR5) · PR5 needs PR3 **deployed to staging** (codegen), not just merged. + +## PR1 — `generic_forms` app core (`generic-forms/01-app`) +- [ ] **T1.1** App skeleton + `Form`/`FormQuestion`/`FormAnswer` + DB constraints + migration + INSTALLED_APPS. Verify: `pytest generic_forms`, `makemigrations --check`. +- [ ] **T1.2** Freeze-on-answer in model (`clean`/`save`/`delete`): type/options/required + delete blocked once answered; label/order/active free. Verify: model tests. +- [ ] **T1.3** `validate_answers()` (returns `{question_id: [errors]}` — same shape as the PR3 wire field) + envelope `wrap/unwrap` (version dispatch). Verify: accept/reject per type. +- [ ] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze-aware readonly), read-only FormAnswerAdmin. Verify: admin tests + manual create-all-types. +- [ ] **▣ CHECKPOINT 1** — full pytest + ruff + mypy green; PR1 opened; human review. + +## PR2 — GraphQL query (`generic-forms/02-graphql-query`) +- [ ] **T2.1** `api/generic_forms/types.py` (FormType, FormQuestionType, enums, options) + `Conference.form(purpose)` (mirror `deadline()`); active-only ordered; null when unconfigured. Verify: `pytest api/generic_forms`. +- [ ] **▣ CHECKPOINT 2** — suite green; schema diff additive; PR2 opened. + +## PR3 — grants backend (`generic-forms/03-grants-backend`) +- [ ] **T3.1** `Grant.form_answer` OneToOne (SET_NULL) + `blank=True` on the 4 required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`); one migration. Columns stay NOT NULL — mutations must never pass `None`. +- [ ] **T3.2** Mutations + tests (TDD): 8 soft fields optional + optional `answers: JSON`; legacy soft-field checks run only when `answers` absent (structured-field validation unchanged on both paths); errors via `answers_errors: JSON` **direct assignment** (dotted paths impossible — verified); FormAnswer `update_or_create` in existing transaction; `None`→`""` coalescing on create; setattr skip-list on update. Named tests: **answers-only payload (exact PR5 shape)**, legacy-shape regression, invalid→atomic rollback, no-form-configured rejected, deadline-closed unchanged, update-no-duplicate. +- [ ] **T3.3** `Grant.formAnswers: JSON|null` on GraphQL type + query test (`me.grant.formAnswers`). +- [ ] **▣ CHECKPOINT 3** — suite green; back-compat verified both directions; PR3 opened; **after merge: deploy to staging (workflow_dispatch) for PR5 codegen**. + +## PR4 — admin display + export (`generic-forms/04-admin`) +- [ ] **T4.1** GrantAdmin readonly Q/A display (`format_html`), empty state, no N+1. Verify: `pytest grants/tests/test_admin.py` + manual. +- [ ] **T4.2** `GrantResource` dynamic columns via `get_export_resource_kwargs` → `__init__` fields append (3.3.9 verified path, incl. export-form preview); historical export unchanged. Verify: resource tests. +- [ ] **▣ CHECKPOINT 4** — suite green; manual CSV export; PR4 opened. + +## PR5 — frontend (`generic-forms/05-frontend`) — start only after PR3 on staging +- [ ] **T5.1** `dynamic-form/` component + fragment + codegen; 6 types via styleguide + InputWrapper (mirror invitation-letter-form); errors from `answersErrors` map; component test. Verify: `pnpm codegen && pnpm test && pnpm build`. +- [ ] **T5.2** New-submission integration: fetch `form(GRANT)`, swap 8 hardcoded inputs for DynamicForm, `answers` in payload (drop legacy 8), **null-form guard blocks submission**, strip legacy validation selections from submit-grant.graphql, prune dead options.ts constants. Verify: pnpm test/build + manual submit. +- [ ] **T5.3** Edit flow: `formAnswers` into my-grant.graphql, prefill DynamicForm, strip legacy selections from edit documents. Verify: pnpm build + manual edit. +- [ ] **▣ CHECKPOINT 5 — FINAL** — spec §11 walked one-by-one; manual E2E (author → submit → edit → admin → export); **ops: GRANT form created in staging + production admin BEFORE merge; cutover before grants open**; follow-up ticketed as TWO PRs (frontend strip → soak → backend input removal). From ed1271705087d32ede9190e1dec7b6fceb273fd8 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:20:13 +0200 Subject: [PATCH 06/13] Mark PR1 tasks done in generic-forms todo --- tasks/generic-forms/todo.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tasks/generic-forms/todo.md b/tasks/generic-forms/todo.md index 5f09ebaa90..f48a6eb7f6 100644 --- a/tasks/generic-forms/todo.md +++ b/tasks/generic-forms/todo.md @@ -3,12 +3,12 @@ Spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Plan: [plan.md](plan.md) Stack: PR1 → PR2 → PR3 → (PR4 ∥ PR5) · PR5 needs PR3 **deployed to staging** (codegen), not just merged. -## PR1 — `generic_forms` app core (`generic-forms/01-app`) -- [ ] **T1.1** App skeleton + `Form`/`FormQuestion`/`FormAnswer` + DB constraints + migration + INSTALLED_APPS. Verify: `pytest generic_forms`, `makemigrations --check`. -- [ ] **T1.2** Freeze-on-answer in model (`clean`/`save`/`delete`): type/options/required + delete blocked once answered; label/order/active free. Verify: model tests. -- [ ] **T1.3** `validate_answers()` (returns `{question_id: [errors]}` — same shape as the PR3 wire field) + envelope `wrap/unwrap` (version dispatch). Verify: accept/reject per type. -- [ ] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze-aware readonly), read-only FormAnswerAdmin. Verify: admin tests + manual create-all-types. -- [ ] **▣ CHECKPOINT 1** — full pytest + ruff + mypy green; PR1 opened; human review. +## PR1 — `generic_forms` app core (`generic-forms/01-app`) — **PR #4705** +- [x] **T1.1** App skeleton + `Form`/`FormQuestion`/`FormAnswer` + DB constraints + migration + INSTALLED_APPS. (b885f5d0e) +- [x] **T1.2** Freeze-on-answer in model: type/options/required/form + delete blocked once answered (pre_delete signal); label/order/active free. (3dfe2ee76) +- [x] **T1.3** `validate_answers()` + envelope `wrap/unwrap`. (43bb587d5) +- [x] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze via model validation errors — readonly deviation documented in plan), read-only FormAnswerAdmin incl. delete block. (f9c2a273d) +- [x] **▣ CHECKPOINT 1** — 48 app tests, full suite 1191 green, ruff clean; adversarial review (3 lenses) applied (455525748); PR #4705 open. **Human review pending. Manual admin eyeball pending.** ## PR2 — GraphQL query (`generic-forms/02-graphql-query`) - [ ] **T2.1** `api/generic_forms/types.py` (FormType, FormQuestionType, enums, options) + `Conference.form(purpose)` (mirror `deadline()`); active-only ordered; null when unconfigured. Verify: `pytest api/generic_forms`. From 6023358ab2bcb5781444412f58653978a7138317 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:27:22 +0200 Subject: [PATCH 07/13] Tighten test assertions to exact error messages Replace substring checks (e.g. asserting "5" appears somewhere in the error) with exact error-dict equality and distinctive pytest.raises match phrases, so tests assert the actual behavior and catch stray extra errors. --- backend/generic_forms/tests/test_models.py | 26 +++++++------ backend/generic_forms/tests/test_services.py | 40 +++++++++++--------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/backend/generic_forms/tests/test_models.py b/backend/generic_forms/tests/test_models.py index 2235ece659..2a66343414 100644 --- a/backend/generic_forms/tests/test_models.py +++ b/backend/generic_forms/tests/test_models.py @@ -54,7 +54,7 @@ def test_question_type_is_frozen_once_form_has_answers(): question = _answered_question(question_type=FormQuestion.QuestionType.TEXT) question.question_type = FormQuestion.QuestionType.TEXTAREA - with pytest.raises(ValidationError, match="question_type"): + with pytest.raises(ValidationError, match="cannot be changed: question_type"): question.save() @@ -65,7 +65,7 @@ def test_options_are_frozen_once_form_has_answers(): ) question.options = [{"id": "b", "label": "B"}] - with pytest.raises(ValidationError, match="options"): + with pytest.raises(ValidationError, match="cannot be changed: options"): question.save() @@ -73,7 +73,7 @@ def test_required_is_frozen_once_form_has_answers(): question = _answered_question(required=False) question.required = True - with pytest.raises(ValidationError, match="required"): + with pytest.raises(ValidationError, match="cannot be changed: required"): question.save() @@ -81,7 +81,7 @@ def test_question_cannot_move_to_another_form_once_answered(): question = _answered_question() question.form = FormFactory() - with pytest.raises(ValidationError, match="form"): + with pytest.raises(ValidationError, match="cannot be changed: form"): question.save() @@ -110,7 +110,9 @@ def test_label_description_order_active_stay_editable_once_answered(): def test_question_cannot_be_deleted_once_form_has_answers(): question = _answered_question() - with pytest.raises(ValidationError, match="deleted"): + with pytest.raises( + ValidationError, match="cannot be deleted because the form already has answers" + ): question.delete() @@ -144,12 +146,14 @@ def test_new_question_can_be_saved_with_an_explicit_pk(): def test_queryset_delete_cannot_remove_questions_from_an_answered_form(): question = _answered_question() - with pytest.raises(ValidationError, match="deleted"): + with pytest.raises( + ValidationError, match="cannot be deleted because the form already has answers" + ): FormQuestion.objects.filter(pk=question.pk).delete() def test_select_options_must_be_a_list_of_id_label_dicts(): - with pytest.raises(ValidationError, match="options"): + with pytest.raises(ValidationError, match="Every option must be"): FormQuestionFactory( question_type=FormQuestion.QuestionType.SELECT, options=["vegan", 42], @@ -157,12 +161,12 @@ def test_select_options_must_be_a_list_of_id_label_dicts(): def test_select_options_cannot_be_empty(): - with pytest.raises(ValidationError, match="options"): + with pytest.raises(ValidationError, match="non-empty list of options"): FormQuestionFactory(question_type=FormQuestion.QuestionType.SELECT, options=[]) def test_option_ids_must_be_unique(): - with pytest.raises(ValidationError, match="options"): + with pytest.raises(ValidationError, match="Option ids must be unique"): FormQuestionFactory( question_type=FormQuestion.QuestionType.MULTI_SELECT, options=[{"id": "a", "label": "A"}, {"id": "a", "label": "Again"}], @@ -170,7 +174,7 @@ def test_option_ids_must_be_unique(): def test_non_choice_questions_cannot_have_options(): - with pytest.raises(ValidationError, match="options"): + with pytest.raises(ValidationError, match="Only select questions can have options"): FormQuestionFactory( question_type=FormQuestion.QuestionType.TEXT, options=[{"id": "a", "label": "A"}], @@ -181,7 +185,7 @@ def test_form_conference_and_purpose_are_frozen_once_answered(): answer = FormAnswerFactory(form__purpose=Form.Purpose.GRANT) answer.form.purpose = Form.Purpose.GENERIC - with pytest.raises(ValidationError, match="purpose"): + with pytest.raises(ValidationError, match="cannot be changed: purpose"): answer.form.save() diff --git a/backend/generic_forms/tests/test_services.py b/backend/generic_forms/tests/test_services.py index 948db85247..7caa0cb95f 100644 --- a/backend/generic_forms/tests/test_services.py +++ b/backend/generic_forms/tests/test_services.py @@ -46,7 +46,7 @@ def test_missing_required_answer_is_an_error(): errors = validate_answers(form, {}) - assert "required" in errors[str(question.pk)][0].lower() + assert errors == {str(question.pk): ["This question is required."]} def test_empty_string_fails_required(): @@ -55,7 +55,7 @@ def test_empty_string_fails_required(): errors = validate_answers(form, {str(question.pk): ""}) - assert "required" in errors[str(question.pk)][0].lower() + assert errors == {str(question.pk): ["This question is required."]} def test_optional_question_can_be_omitted(): @@ -77,7 +77,7 @@ def test_unknown_question_id_is_an_error(): errors = validate_answers(form, {"9999": "hello"}) - assert "9999" in errors + assert errors == {"9999": ["Unknown or inactive question."]} def test_inactive_question_id_is_an_error(): @@ -86,7 +86,7 @@ def test_inactive_question_id_is_an_error(): errors = validate_answers(form, {str(question.pk): "hello"}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Unknown or inactive question."]} def test_text_answer_must_be_a_string(): @@ -95,7 +95,7 @@ def test_text_answer_must_be_a_string(): errors = validate_answers(form, {str(question.pk): 123}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Invalid value: expected text."]} def test_text_answer_respects_max_length(): @@ -104,7 +104,7 @@ def test_text_answer_respects_max_length(): errors = validate_answers(form, {str(question.pk): "too long"}) - assert "5" in errors[str(question.pk)][0] + assert errors == {str(question.pk): ["Cannot be longer than 5 characters."]} def test_select_answer_must_be_a_known_option(): @@ -113,7 +113,7 @@ def test_select_answer_must_be_a_known_option(): errors = validate_answers(form, {str(question.pk): "carnivore"}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Invalid option."]} def test_multi_select_must_be_a_list(): @@ -124,7 +124,9 @@ def test_multi_select_must_be_a_list(): errors = validate_answers(form, {str(question.pk): "vegan"}) - assert str(question.pk) in errors + assert errors == { + str(question.pk): ["Invalid value: expected a list of option ids."] + } def test_multi_select_rejects_non_string_items_without_crashing(): @@ -135,7 +137,9 @@ def test_multi_select_rejects_non_string_items_without_crashing(): errors = validate_answers(form, {str(question.pk): [["vegan"]]}) - assert str(question.pk) in errors + assert errors == { + str(question.pk): ["Invalid value: expected a list of option ids."] + } def test_non_dict_answers_return_a_global_error(): @@ -154,7 +158,7 @@ def test_multi_select_rejects_a_single_unknown_item(): errors = validate_answers(form, {str(question.pk): ["vegan", "carnivore"]}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Invalid options: carnivore."]} def test_boolean_answer_must_be_a_bool(): @@ -163,7 +167,7 @@ def test_boolean_answer_must_be_a_bool(): errors = validate_answers(form, {str(question.pk): "yes"}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Invalid value: expected true or false."]} def test_url_answer_must_be_a_valid_url(): @@ -172,7 +176,7 @@ def test_url_answer_must_be_a_valid_url(): errors = validate_answers(form, {str(question.pk): "not a url"}) - assert str(question.pk) in errors + assert errors == {str(question.pk): ["Invalid URL."]} def test_multiple_errors_are_collected_per_call(): @@ -182,8 +186,10 @@ def test_multiple_errors_are_collected_per_call(): errors = validate_answers(form, {str(boolean.pk): "yes"}) - assert str(required.pk) in errors - assert str(boolean.pk) in errors + assert errors == { + str(required.pk): ["This question is required."], + str(boolean.pk): ["Invalid value: expected true or false."], + } def test_wrap_and_unwrap_answers_round_trip(): @@ -205,9 +211,9 @@ def test_unwrap_answers_treats_empty_envelope_as_no_answers(): def test_unwrap_answers_rejects_malformed_envelopes(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Malformed answers envelope"): unwrap_answers(["not", "a", "dict"]) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="missing answers map"): unwrap_answers({"version": 1}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="missing answers map"): unwrap_answers({"version": 1, "answers": "not a dict"}) From 7581cfa8c31497059902b037244b99c5c9987807 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:31:27 +0200 Subject: [PATCH 08/13] Drop over-specific explicit-pk save test --- backend/generic_forms/tests/test_models.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/backend/generic_forms/tests/test_models.py b/backend/generic_forms/tests/test_models.py index 2a66343414..103ae87b39 100644 --- a/backend/generic_forms/tests/test_models.py +++ b/backend/generic_forms/tests/test_models.py @@ -130,19 +130,6 @@ def test_new_question_can_be_added_to_answered_form(): FormQuestionFactory(form=question.form) -def test_new_question_can_be_saved_with_an_explicit_pk(): - form = FormFactory() - - FormQuestion( - pk=987654, - form=form, - label="Explicit pk", - question_type=FormQuestion.QuestionType.TEXT, - ).save() - - assert FormQuestion.objects.filter(pk=987654).exists() - - def test_queryset_delete_cannot_remove_questions_from_an_answered_form(): question = _answered_question() From 88b4c51442140193ec4ee6679de77aedd87f6483 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:35:06 +0200 Subject: [PATCH 09/13] Simplify generic_forms internals - validate_answers: plain dict instead of defaultdict + trailing filter comprehension (empty entries no longer created then dropped) - extract _is_valid_option predicate from the option-shape mega-boolean - reuse a module-level URLValidator; drop redundant map(str, ...) on already-validated strings; idiomatic empty-dict check in unwrap - FormAnswer admin: select related form__conference/user (Form.__str__ renders the conference name) --- backend/generic_forms/admin.py | 2 ++ backend/generic_forms/models.py | 31 +++++++++++++++++-------------- backend/generic_forms/services.py | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/backend/generic_forms/admin.py b/backend/generic_forms/admin.py index 279bd6d4db..07e866d05d 100644 --- a/backend/generic_forms/admin.py +++ b/backend/generic_forms/admin.py @@ -45,6 +45,8 @@ def get_readonly_fields(self, request, obj=None): class FormAnswerAdmin(admin.ModelAdmin): list_display = ("form", "user", "created") list_filter = ("form__conference", "form__purpose") + # Form.__str__ renders the conference name + list_select_related = ("form__conference", "user") def has_add_permission(self, request): return False diff --git a/backend/generic_forms/models.py b/backend/generic_forms/models.py index 1c55a6ab14..85b7cdc703 100644 --- a/backend/generic_forms/models.py +++ b/backend/generic_forms/models.py @@ -66,6 +66,16 @@ class Meta: ] +def _is_valid_option(option) -> bool: + return ( + isinstance(option, dict) + and isinstance(option.get("id"), str) + and option["id"] != "" + and isinstance(option.get("label"), str) + and option["label"] != "" + ) + + class FormQuestion(TimeStampedModel): class QuestionType(models.TextChoices): TEXT = "text", _("Text") @@ -123,20 +133,13 @@ def _validate_options(self): {"options": "Select questions need a non-empty list of options."} ) - for option in self.options: - if ( - not isinstance(option, dict) - or not isinstance(option.get("id"), str) - or not option["id"] - or not isinstance(option.get("label"), str) - or not option["label"] - ): - raise ValidationError( - { - "options": 'Every option must be {"id": "...", ' - '"label": "..."} with non-empty strings.' - } - ) + if not all(_is_valid_option(option) for option in self.options): + raise ValidationError( + { + "options": 'Every option must be {"id": "...", ' + '"label": "..."} with non-empty strings.' + } + ) ids = [option["id"] for option in self.options] if len(ids) != len(set(ids)): diff --git a/backend/generic_forms/services.py b/backend/generic_forms/services.py index 317d8e9131..5dcb2717b1 100644 --- a/backend/generic_forms/services.py +++ b/backend/generic_forms/services.py @@ -1,5 +1,3 @@ -from collections import defaultdict - from django.core.exceptions import ValidationError from django.core.validators import URLValidator @@ -7,6 +5,8 @@ ANSWERS_VERSION = 1 +_validate_url = URLValidator() + def wrap_answers(answers: dict) -> dict: return {"version": ANSWERS_VERSION, "answers": answers} @@ -15,7 +15,7 @@ def wrap_answers(answers: dict) -> dict: def unwrap_answers(envelope: dict) -> dict: if not isinstance(envelope, dict): raise ValueError("Malformed answers envelope.") - if envelope == {}: + if not envelope: # a FormAnswer created without going through wrap_answers return {} version = envelope.get("version") @@ -34,28 +34,28 @@ def validate_answers(form: Form, answers: dict) -> dict[str, list[str]]: if not isinstance(answers, dict): return {"__all__": ["Invalid answers format."]} - errors: dict[str, list[str]] = defaultdict(list) + errors: dict[str, list[str]] = {} questions = { str(question.pk): question for question in form.questions.filter(active=True) } for question_id in answers: if question_id not in questions: - errors[question_id].append("Unknown or inactive question.") + errors[question_id] = ["Unknown or inactive question."] for question_id, question in questions.items(): value = answers.get(question_id) if value is None or value == "" or value == []: if question.required: - errors[question_id].append("This question is required.") + errors[question_id] = ["This question is required."] continue - errors[question_id].extend(_validate_value(question, value)) + messages = _validate_value(question, value) + if messages: + errors[question_id] = messages - return { - question_id: messages for question_id, messages in errors.items() if messages - } + return errors def _validate_value(question: FormQuestion, value) -> list[str]: @@ -69,7 +69,7 @@ def _validate_value(question: FormQuestion, value) -> list[str]: return [f"Cannot be longer than {question.max_length} characters."] if question_type == types.URL: try: - URLValidator()(value) + _validate_url(value) except ValidationError: return ["Invalid URL."] return [] @@ -86,7 +86,7 @@ def _validate_value(question: FormQuestion, value) -> list[str]: return ["Invalid value: expected a list of option ids."] invalid = [item for item in value if item not in _option_ids(question)] if invalid: - return ["Invalid options: " + ", ".join(map(str, invalid)) + "."] + return [f"Invalid options: {', '.join(invalid)}."] return [] if question_type == types.BOOLEAN: From c21b4bcd34b049647ee39e15724fd1f75acf6c74 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:48:51 +0200 Subject: [PATCH 10/13] Expose conference forms over GraphQL conference.form(purpose) returns the form configured for the given purpose (null when unconfigured) with its active questions in order: label, description, type, required, maxLength and select options. Query-side only; submitting answers lands with the grants integration. --- backend/api/conferences/types.py | 6 + backend/api/generic_forms/__init__.py | 0 backend/api/generic_forms/tests/__init__.py | 0 .../generic_forms/tests/test_form_query.py | 131 ++++++++++++++++++ backend/api/generic_forms/types.py | 43 ++++++ 5 files changed, 180 insertions(+) create mode 100644 backend/api/generic_forms/__init__.py create mode 100644 backend/api/generic_forms/tests/__init__.py create mode 100644 backend/api/generic_forms/tests/test_form_query.py create mode 100644 backend/api/generic_forms/types.py diff --git a/backend/api/conferences/types.py b/backend/api/conferences/types.py index c75d451ae6..e5a7238c8f 100644 --- a/backend/api/conferences/types.py +++ b/backend/api/conferences/types.py @@ -12,6 +12,8 @@ from strawberry import ID from api.cms.types import FAQ, Menu from api.events.types import Event +from api.generic_forms.types import Form as GenericForm +from api.generic_forms.types import FormPurpose from api.languages.types import Language from api.pretix.query import get_conference_tickets, get_voucher from api.pretix.types import TicketItem, Voucher @@ -197,6 +199,10 @@ def is_voting_closed(self, info: Info) -> bool: def deadline(self, info: Info, type: str) -> Deadline | None: return self.deadlines.filter(type=type).first() + @strawberry.field + def form(self, info: Info, purpose: FormPurpose) -> GenericForm | None: + return self.forms.filter(purpose=purpose).first() + @strawberry.field def audience_levels(self, info: Info) -> list[AudienceLevel]: return self.audience_levels.all() diff --git a/backend/api/generic_forms/__init__.py b/backend/api/generic_forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/api/generic_forms/tests/__init__.py b/backend/api/generic_forms/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/api/generic_forms/tests/test_form_query.py b/backend/api/generic_forms/tests/test_form_query.py new file mode 100644 index 0000000000..83a7099f7b --- /dev/null +++ b/backend/api/generic_forms/tests/test_form_query.py @@ -0,0 +1,131 @@ +import pytest + +from conferences.tests.factories import ConferenceFactory +from generic_forms.models import Form, FormQuestion +from generic_forms.tests.factories import FormFactory, FormQuestionFactory + +pytestmark = pytest.mark.django_db + + +def _query_form(graphql_client, conference, purpose="GRANT"): + query = """query($code: String!, $purpose: FormPurpose!) { + conference(code: $code) { + form(purpose: $purpose) { + id + name + questions { + id + label + description + questionType + required + maxLength + options { + id + label + } + } + } + } + }""" + + return graphql_client.query( + query, variables={"code": conference.code, "purpose": purpose} + ) + + +def test_form_is_none_when_not_configured(graphql_client): + conference = ConferenceFactory() + + result = _query_form(graphql_client, conference) + + assert result["data"]["conference"]["form"] is None + + +def test_form_is_none_when_only_another_purpose_is_configured(graphql_client): + form = FormFactory(purpose=Form.Purpose.GENERIC) + + result = _query_form(graphql_client, form.conference, purpose="GRANT") + + assert result["data"]["conference"]["form"] is None + + +def test_form_belongs_to_the_requested_conference(graphql_client): + FormFactory(purpose=Form.Purpose.GRANT, name="Other conference form") + conference = ConferenceFactory() + + result = _query_form(graphql_client, conference) + + assert result["data"]["conference"]["form"] is None + + +def test_form_with_questions(graphql_client): + form = FormFactory(purpose=Form.Purpose.GRANT, name="Grant form") + question = FormQuestionFactory( + form=form, + label="Why do you want to attend?", + description="Tell us more", + question_type=FormQuestion.QuestionType.TEXTAREA, + required=True, + max_length=500, + order=0, + ) + + result = _query_form(graphql_client, form.conference) + + data = result["data"]["conference"]["form"] + assert data["id"] == str(form.id) + assert data["name"] == "Grant form" + assert data["questions"] == [ + { + "id": str(question.id), + "label": "Why do you want to attend?", + "description": "Tell us more", + "questionType": "TEXTAREA", + "required": True, + "maxLength": 500, + "options": [], + } + ] + + +def test_select_question_options_are_exposed(graphql_client): + form = FormFactory(purpose=Form.Purpose.GRANT) + FormQuestionFactory( + form=form, + question_type=FormQuestion.QuestionType.SELECT, + options=[ + {"id": "vegan", "label": "Vegan"}, + {"id": "veggie", "label": "Veggie"}, + ], + ) + + result = _query_form(graphql_client, form.conference) + + assert result["data"]["conference"]["form"]["questions"][0]["options"] == [ + {"id": "vegan", "label": "Vegan"}, + {"id": "veggie", "label": "Veggie"}, + ] + + +def test_questions_follow_the_configured_order(graphql_client): + form = FormFactory(purpose=Form.Purpose.GRANT) + second = FormQuestionFactory(form=form, order=1) + first = FormQuestionFactory(form=form, order=0) + third = FormQuestionFactory(form=form, order=2) + + result = _query_form(graphql_client, form.conference) + + ids = [q["id"] for q in result["data"]["conference"]["form"]["questions"]] + assert ids == [str(first.id), str(second.id), str(third.id)] + + +def test_inactive_questions_are_hidden(graphql_client): + form = FormFactory(purpose=Form.Purpose.GRANT) + active = FormQuestionFactory(form=form, active=True) + FormQuestionFactory(form=form, active=False) + + result = _query_form(graphql_client, form.conference) + + ids = [q["id"] for q in result["data"]["conference"]["form"]["questions"]] + assert ids == [str(active.id)] diff --git a/backend/api/generic_forms/types.py b/backend/api/generic_forms/types.py new file mode 100644 index 0000000000..37c9af9430 --- /dev/null +++ b/backend/api/generic_forms/types.py @@ -0,0 +1,43 @@ +import strawberry + +from api.context import Info +from generic_forms.models import Form as FormModel +from generic_forms.models import FormQuestion as FormQuestionModel + +FormPurpose = strawberry.enum(FormModel.Purpose, name="FormPurpose") +FormQuestionType = strawberry.enum( + FormQuestionModel.QuestionType, name="FormQuestionType" +) + + +@strawberry.type +class FormQuestionOption: + id: str + label: str + + +@strawberry.type +class FormQuestion: + id: strawberry.ID + label: str + description: str + question_type: FormQuestionType + required: bool + max_length: int | None + + @strawberry.field + def options(self, info: Info) -> list[FormQuestionOption]: + return [ + FormQuestionOption(id=option["id"], label=option["label"]) + for option in self.options + ] + + +@strawberry.type +class Form: + id: strawberry.ID + name: str + + @strawberry.field + def questions(self, info: Info) -> list[FormQuestion]: + return self.questions.filter(active=True) From 1c9fb3490c5d4b8f372ae53759479c1ca534a785 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Thu, 6 Aug 2026 15:49:22 +0200 Subject: [PATCH 11/13] Mark PR2 tasks done in generic-forms todo --- tasks/generic-forms/todo.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/generic-forms/todo.md b/tasks/generic-forms/todo.md index f48a6eb7f6..39bb62d554 100644 --- a/tasks/generic-forms/todo.md +++ b/tasks/generic-forms/todo.md @@ -10,9 +10,9 @@ Stack: PR1 → PR2 → PR3 → (PR4 ∥ PR5) · PR5 needs PR3 **deployed to stag - [x] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze via model validation errors — readonly deviation documented in plan), read-only FormAnswerAdmin incl. delete block. (f9c2a273d) - [x] **▣ CHECKPOINT 1** — 48 app tests, full suite 1191 green, ruff clean; adversarial review (3 lenses) applied (455525748); PR #4705 open. **Human review pending. Manual admin eyeball pending.** -## PR2 — GraphQL query (`generic-forms/02-graphql-query`) -- [ ] **T2.1** `api/generic_forms/types.py` (FormType, FormQuestionType, enums, options) + `Conference.form(purpose)` (mirror `deadline()`); active-only ordered; null when unconfigured. Verify: `pytest api/generic_forms`. -- [ ] **▣ CHECKPOINT 2** — suite green; schema diff additive; PR2 opened. +## PR2 — GraphQL query (`generic-forms/02-graphql-query`) — **PR #4707** +- [x] **T2.1** `api/generic_forms/types.py` (Form, FormQuestion, FormQuestionOption, FormPurpose/FormQuestionType enums) + `Conference.form(purpose)`; active-only ordered; null when unconfigured. 7 tests. +- [x] **▣ CHECKPOINT 2** — full suite 1197 green; additive-only schema change; PR #4707 open (stacked on #4705). ## PR3 — grants backend (`generic-forms/03-grants-backend`) - [ ] **T3.1** `Grant.form_answer` OneToOne (SET_NULL) + `blank=True` on the 4 required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`); one migration. Columns stay NOT NULL — mutations must never pass `None`. From d2b4b608f9604d66c30584622c2e7864d9216ec2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 7 Aug 2026 04:38:06 +0200 Subject: [PATCH 12/13] Remove spec and task docs from the PR --- specs/generic-form-system.md | 303 --------------------------------- tasks/generic-forms/plan.md | 316 ----------------------------------- tasks/generic-forms/todo.md | 32 ---- 3 files changed, 651 deletions(-) delete mode 100644 specs/generic-form-system.md delete mode 100644 tasks/generic-forms/plan.md delete mode 100644 tasks/generic-forms/todo.md diff --git a/specs/generic-form-system.md b/specs/generic-form-system.md deleted file mode 100644 index da980eba3e..0000000000 --- a/specs/generic-form-system.md +++ /dev/null @@ -1,303 +0,0 @@ -# Spec: Generic Form System - -Status: Approved — ready for planning -Source: Notion draft "Generic Form system" (exported HTML in repo root) + clarifying Q&A -Author: generated via spec-driven-development - ---- - -## 1. Objective - -Build a generic, per-conference configurable form system so organizers can change the questions asked in recurring flows (grants, CFP, visa, feedback) **without backend or frontend code changes**. Today every question is a hardcoded model column (`Grant`, `Submission`) or an external Google Form; changing questions for a new conference edition requires coordinated BE + FE work and migrations. - -**First consumer (this spec's scope): the grant application form.** The engine is built generically; grants is the first flow wired to it. CFP, visa, and feedback forms are explicitly future slices. - -**Target users:** -- *Organizers* — author/edit form questions per conference in Django admin. -- *Attendees/applicants* — fill forms on the Next.js frontend. -- *Maintainers* — stop writing migrations + form components for every question change. - -**Success looks like:** an organizer can add, reword, reorder, or deactivate a grant-form question for the next conference entirely from Django admin, and the frontend renders and validates it with zero code changes. - -### Decisions already made (via Q&A) - -1. **MVP integration target: grants** (biggest pain; `Grant` has ~20 hardcoded answer columns). -2. **Data model: hybrid** — `Form`/`FormQuestion` as normal models (admin-authorable), answers stored as a single `FormAnswer` row per submission with a `JSONField` mapping `question_id → value`. No per-question answer rows. -3. **Versioning: freeze-on-answer** — a question's semantic fields (type, options, required) become immutable once any answer exists for its form. Changes happen by deactivating questions and adding new ones (or cloning the form for a new conference). No snapshot or version-row machinery. -4. **Authoring UI: Django admin** — inline `FormQuestion` editing under `Form`. No custom-admin/Astro builder in this slice. -5. **Load-bearing grant fields stay as `Grant` columns** (confirmed). Fields that drive business logic — `grant_type` (reimbursement categories), `departure_country`/`nationality` (`country_type` derivation, visa), `departure_city`, `needs_funds_for_travel`, `need_visa`, `need_accommodation` — remain structured columns on `Grant`, as do `full_name`/`name`. The *soft* questions moving into the generic form are exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Corrected during planning: socials/website do NOT move — the grant form's social inputs are `participant_*` fields handled via `PublicProfileCard`/`Participant` upsert, not Grant columns; Grant's own social columns are already unused by the current flow.) This avoids a question→field mapping layer in the MVP. -6. **English only** — no multi-lingual labels/options (confirmed). -7. **Options-as-JSON admin UX**: raw JSON widget is acceptable — no custom widget (confirmed). -8. **Grant admin export includes dynamic answers in this slice** (confirmed). The existing `GrantResource` (django-import-export, `grants/admin.py`) exports several soft-question columns today; those move to dynamic-answer columns — one column per question of the conference's grant form (the export is already single-conference via `before_export`). -9. **`purpose` enum values for cfp/visa/feedback are added when those slices land**, not preemptively (confirmed). - -### Assumptions I'm making (correct before approval if wrong) - -1. **No data migration of historical grants.** Old `Grant` columns stay populated and readable for past conferences; new conferences write soft answers to `FormAnswer` only. Legacy columns become nullable/blank-able but are **not dropped** in this slice. -2. **One `FormAnswer` per (form, user).** Matches the existing one-grant-per-user-per-conference constraint. Multi-response generic forms (e.g. anonymous feedback) are future work. -3. **Question labels/descriptions are editable even after answers exist** (typo fixes); only `question_type`, `options`, and `required` freeze. Deletion is blocked once answered — deactivate instead. -4. **New Django app named `generic_forms`** (avoids collision/confusion with `django.forms` and `wagtail.contrib.forms`, which is installed but unused). -5. **Select options live in a `JSONField` on `FormQuestion`** (list of `{id, label}`), not a third model — Django admin can't nest inlines two levels deep, and options-as-JSON keeps authoring on one page. -6. **No file-upload question type in MVP** — it requires extending `files_upload.File.Type`, size limits, and upload permissions. Listed as future work. -7. **No conditional/branching questions in MVP.** - ---- - -## 2. Scope - -### In scope - -- New `generic_forms` Django app: `Form`, `FormQuestion`, `FormAnswer` models + migrations + admin. -- Question types: `text` (single line), `textarea`, `select`, `multi_select`, `boolean`, `url`. -- Server-side answer validation (required, type, option membership, max length, URL format) following the existing `BaseErrorType` pattern. -- GraphQL: query a conference's form by purpose (id, name, ordered active questions with labels/options); mutation to submit/update answers is folded into the existing grant mutations (see §5). -- Grants integration: `sendGrant`/`updateGrant` accept an `answers` input, validate against the conference's grant form, persist a `FormAnswer` linked from `Grant`. -- Frontend: a reusable `DynamicForm` component (styleguide inputs, `react-use-form-state`) rendering questions by type; grant form page renders its soft-question sections dynamically. -- Django admin: grant admin displays the applicant's dynamic answers read-only alongside the structured fields. -- Grant admin export: `GrantResource` gains one column per question of the conference's grant form, populated from the linked `FormAnswer`; legacy soft-question columns stay for historical exports. -- Freeze-on-answer enforcement at the model layer (not just admin). - -### Out of scope (explicitly NOT in this slice) - -- CFP/Submission, visa, and feedback form integrations (engine supports `purpose` values for them, but no product wiring). -- Migrating historical `Grant` answer data into `FormAnswer`; dropping legacy `Grant` columns. -- Custom-admin (Astro) form-builder UI; Wagtail integration. -- File-upload, date, number, or conditional question types. -- Anonymous / multi-response forms. -- Generic "form submitted" confirmation email plumbing (draft's idea — good future win, not now; grants keeps its existing notification path). -- Changes to Pretix, Stripe, or the reimbursement flow. -- Profile-based prefill of dynamic answers (today `ageGroup` prefills from `user.dateBirth` and `gender` from `user.gender`; the generic engine has no per-question semantics, so these prefills are dropped — small accepted UX regression). - ---- - -## 3. Tech stack - -- **Backend:** Django 5.x (existing), PostgreSQL, Strawberry GraphQL. No new Python dependencies expected. -- **Language:** English only — plain `CharField`/`TextField` for labels, descriptions, option labels. No `I18nCharField`/`I18nTextField`. -- **Frontend:** Next.js (existing), TypeScript, Apollo Client with codegen (`pnpm codegen`), `react-use-form-state` (corrected during planning: `react-hook-form` is in package.json but has zero usages in the codebase — every existing form, including the modern invitation-letter form, uses `react-use-form-state`; the new component follows the actual in-repo pattern), `@python-italia/pycon-styleguide` inputs. -- **Admin:** stock Django admin with `TabularInline`/`StackedInline`. - ---- - -## 4. Data model - -```python -# backend/generic_forms/models.py -class Form(TimeStampedModel): - class Purpose(models.TextChoices): - GRANT = "grant", _("Grant") - GENERIC = "generic", _("Generic") # cfp/visa/feedback added in later slices - - conference = models.ForeignKey("conferences.Conference", on_delete=models.CASCADE, - related_name="forms") - purpose = models.CharField(max_length=32, choices=Purpose.choices) - name = models.CharField(max_length=200) - # constraint: at most one form per (conference, purpose) when purpose != GENERIC - - -class FormQuestion(TimeStampedModel): - class QuestionType(models.TextChoices): - TEXT = "text" - TEXTAREA = "textarea" - SELECT = "select" - MULTI_SELECT = "multi_select" - BOOLEAN = "boolean" - URL = "url" - - form = models.ForeignKey(Form, on_delete=models.CASCADE, related_name="questions") - label = models.CharField(max_length=300) - description = models.TextField(blank=True) - question_type = models.CharField(max_length=32, choices=QuestionType.choices) - options = models.JSONField(blank=True, default=list) - # options item shape: {"id": "vegan", "label": "Vegan"} - required = models.BooleanField(default=False) - max_length = models.PositiveIntegerField(null=True, blank=True) - order = models.PositiveIntegerField(default=0) - active = models.BooleanField(default=True) # deactivate instead of delete once answered - - -class FormAnswer(TimeStampedModel): - form = models.ForeignKey(Form, on_delete=models.PROTECT, related_name="answers") - user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - answers = models.JSONField(default=dict) - # Versioned envelope so the structure can evolve without guessing: - # {"version": 1, "answers": {"": value}} - # version 1 value types by question_type: - # text/textarea/url → str, select → option id (str), - # multi_select → list[str] of option ids, boolean → bool - # Readers dispatch on "version"; writers always write the current version. - # (GraphQL input stays the flat {question_id: value} map — the envelope is - # a storage concern; the mutation wraps it on persist.) - - class Meta: - constraints = [models.UniqueConstraint(fields=["form", "user"], - name="unique_form_answer_per_user")] -``` - -**Grant link:** `Grant.form_answer = models.OneToOneField("generic_forms.FormAnswer", null=True, blank=True, on_delete=models.SET_NULL)`. Soft-question columns on `Grant` become `blank=True` (kept for historical data). - -**Freeze-on-answer rule (model layer):** `FormQuestion.clean()`/`save()` raise if `question_type`, `options`, or `required` change while `self.form.answers.exists()`; deletion is blocked via a `pre_delete` signal (covers queryset deletes too). `label`/`description`/`order`/`active` stay editable. `Form.conference`/`purpose` freeze the same way. In admin the rule surfaces as validation errors on the inline (not readonly fields — inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); the model is the enforcement point. Question `options` are shape-validated at authoring (list of `{id, label}` string pairs, unique ids, required for select types, forbidden otherwise). - -**Answer validation (single source of truth):** a `validate_answers(form, answers: dict) -> dict[str, list[str]]` service in `generic_forms/` used by the GraphQL layer: unknown/inactive question ids rejected, required enforced, per-type checks (option membership incl. every item of multi_select, `URLValidator` for url, `max_length` for text types, bool type check). - ---- - -## 5. API design (GraphQL) - -Follows the newer one-mutation-per-file pattern and the `api/visa/mutations/request_invitation_letter.py` validation style. - -**Query** — extend the existing `Conference` type: - -```graphql -conference(code: "pycon2026") { - form(purpose: GRANT) { # null if no form configured - id - name - questions { # active only, ordered - id - label - description - questionType - required - maxLength - options { id label } - } - } -} -``` - -**Mutations** — no standalone `submitFormAnswers` in this slice. `sendGrant` / `updateGrant` inputs gain an optional `answers: JSON` (map of question id → value). The mutation: -1. Keeps its existing deadline gating unchanged (`non_field_errors: "The grants form is not open!"` via `Conference.is_grants_open`) — no `FormNotAvailable` union member (changing the deadline-closed response shape would break the deployed frontend; decided during planning). -2. If `answers` is provided but the conference has no `GRANT` form, rejects with a clear error. If the form exists, runs `validate_answers`; failures are returned in a dedicated `answersErrors: JSON` field on `GrantErrors` mapping `question_id → [messages]`. (Dotted dynamic paths like `answers.` cannot serialize through the statically-typed error classes — verified during planning; the in-repo dotted-path precedent, `materials.0.url` in `api/submissions`, works only because its container field is statically declared.) -3. Persists `FormAnswer` (create or update), wrapping the input map into the versioned envelope (`{"version": 1, "answers": {...}}`), and links it to the `Grant` in the same transaction. -4. The 8 legacy soft input fields become optional; legacy-shape submissions (soft fields, no `answers`) keep working unchanged until the frontend cutover, then get removed in a post-deploy follow-up. - -**Grant type** — exposes `formAnswers: JSON | null` (the unwrapped flat map) so the frontend edit flow can prefill the dynamic questions. - -Privacy policy acceptance, Slack notification, and email template lookups keep their current grant-specific wiring — unchanged. - ---- - -## 6. Commands - -All backend commands run inside Docker (per CLAUDE.md). - -| Purpose | Command | -|---|---| -| Run backend tests (new app) | `docker exec pycon-backend-1 uv run pytest generic_forms/tests api/generic_forms -l -s -vvv` | -| Grants integration tests | `docker exec pycon-backend-1 uv run pytest api/grants grants -l -s -vvv` | -| Full suite | `docker exec pycon-backend-1 uv run pytest` | -| Make migrations | `docker exec pycon-backend-1 uv run python manage.py makemigrations generic_forms grants` | -| Migrate | `docker exec pycon-backend-1 uv run python manage.py migrate` | -| Lint / format | `docker exec pycon-backend-1 uv run ruff check` / `uv run ruff format` | -| Type check | `docker exec pycon-backend-1 uv run mypy .` | -| Frontend codegen (after schema change) | `cd frontend && pnpm codegen` | -| Frontend tests / build | `cd frontend && pnpm test` / `pnpm build` | - ---- - -## 7. Project structure - -``` -backend/ - generic_forms/ # NEW app - models.py # Form, FormQuestion, FormAnswer - services.py # validate_answers() - admin.py # Form admin + FormQuestion inline (freeze-aware) - migrations/ - tests/ # model + validation tests, factories - api/ - generic_forms/ # NEW: FormType, FormQuestionType (query side) - types.py - grants/mutations.py # extend sendGrant/updateGrant with answers - grants/ - models.py # + form_answer FK; soft columns → blank=True - admin.py # + read-only answers display - pycon/settings/base.py # + generic_forms in INSTALLED_APPS - -frontend/src/ - components/dynamic-form/ # NEW: renders FormQuestion[] via styleguide inputs - index.tsx - form.graphql # fragment for form + questions - components/grant-form/ # integrate DynamicForm for soft questions -``` - ---- - -## 8. Code style - -Backend follows existing conventions — Ruff (lint + format), mypy clean. Mutation validation mirrors the in-repo pattern: - -```python -@strawberry.input -class SendGrantInput: - conference: strawberry.ID - answers: JSON - ... - - def validate(self, conference: Conference, form: Form) -> GrantErrors | None: - errors = GrantErrors() - if answer_errors := validate_answers(form, self.answers): - # dedicated JSON field: {question_id: [messages]} — dynamic keys - # cannot serialize through the statically-typed error fields - errors.answers_errors = answer_errors - return errors.if_has_errors -``` - -Frontend: `react-use-form-state` + `@python-italia/pycon-styleguide` primitives (mirror `invitation-letter-form.tsx`: `InputWrapper` around each field, `MultiplePartsCard` sections); GraphQL documents co-located with components; **never hand-edit generated files** (`src/types.tsx`, `src/generated/`). - ---- - -## 9. Testing strategy - -- **Framework:** pytest + factory-based fixtures, in-app `tests/` dirs (existing convention). Frontend: existing `pnpm test` setup for the `DynamicForm` component's rendering/validation mapping. -- **Model tests** (`generic_forms/tests/`): freeze-on-answer (type/options/required change and delete blocked once an answer exists; label/order/active edits allowed); unique (form, user) constraint; one-form-per-(conference, purpose) constraint. -- **Validation tests:** each question type's accept/reject cases — required missing, wrong value type, unknown question id, inactive question id, non-member option, multi_select with one bad item, invalid URL, over max_length. -- **API tests** (`api/` tests): query returns only active questions in order; `sendGrant` with valid answers creates `Grant` + linked `FormAnswer` atomically; invalid answers return per-question errors in `answersErrors` and persist nothing; answers-with-no-form-configured is rejected; grants-deadline-closed behavior unchanged from today; an answers-only payload omitting all 8 legacy soft fields succeeds end-to-end (this is the exact post-cutover frontend payload). -- **Export test:** `GrantResource` export of a grant with a linked `FormAnswer` produces one column per form question with the answer values (option ids resolved to labels); grants without `FormAnswer` (historical) still export cleanly. -- **Regression:** full existing grants test suite stays green — legacy columns still accepted for old data paths. -- Every slice lands with its tests; `pytest`, `ruff check`, `mypy .` green before any commit. - ---- - -## 10. Boundaries - -### Always do -- Run backend commands via `docker exec pycon-backend-1 ...` (local venv doesn't work). -- Run `pytest` + `ruff check` + `mypy .` (and `pnpm codegen` after schema changes) before committing. -- Enforce freeze-on-answer in the model, not only in admin. -- Validate answers server-side via `validate_answers` — frontend validation is UX only. -- Keep legacy `Grant` columns readable (admin, exports) for historical conferences. - -### Ask first -- Adding any new dependency (backend or frontend). -- Changing which `Grant` fields count as load-bearing (decision #5) — i.e. moving `grant_type`, country, or `need_*` fields into the form. -- Any data migration touching existing `Grant` rows beyond `blank=True` loosening. -- Adding new values to `files_upload.File.Type` (file-upload question type). -- Schema changes to `Submission`, visa, or notification models. -- Dropping or renaming any existing column. - -### Never -- Drop legacy `Grant` answer columns in this slice. -- Hand-edit generated GraphQL types (`frontend/src/types.tsx`, `*.generated.ts`). -- Store answers as per-question rows (decision: JSON) or bypass `validate_answers` in any mutation. -- Commit secrets; weaken rate-limit/permission classes on mutations. -- Delete or skip failing tests to get green. - ---- - -## 11. Success criteria - -1. Organizer creates a `GRANT` form with questions of every supported type in Django admin, reorders and deactivates questions — no code change needed. -2. Once one answer exists, changing a question's type/options/required or deleting it fails with a clear error in both admin and direct model save; label typo fix still succeeds. -3. `conference.form(purpose: GRANT)` returns the ordered active questions; returns `null` when unconfigured. -4. `sendGrant` with valid `answers` creates `Grant` + linked `FormAnswer` in one transaction; a second submit by the same user for the same conference updates rather than duplicates (existing update path). -5. `sendGrant` with an invalid answer (missing required, bad option, invalid URL) returns per-question errors (`answersErrors` map) and writes nothing; an answers-only payload with no legacy soft fields succeeds. -6. Grant form page on the frontend renders the soft-question sections from the API (verify: add a question in admin → it appears on the page after reload, no deploy of new code). -7. Grant admin shows the applicant's dynamic answers read-only next to structured fields. -8. Grant admin export includes a column per form question with the applicant's answers; exports of historical grants (no `FormAnswer`) still work. -9. Full backend test suite, `ruff check`, `mypy .`, frontend `pnpm build` + `pnpm test` all green. - -## 12. Open questions - -None — all resolved into decisions #7–#9. diff --git a/tasks/generic-forms/plan.md b/tasks/generic-forms/plan.md deleted file mode 100644 index add0215ed1..0000000000 --- a/tasks/generic-forms/plan.md +++ /dev/null @@ -1,316 +0,0 @@ -# Implementation Plan: Generic Form System - -Source spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Mode: plan (read-only, no code changed) -Structure: **5 stacked PRs** — each PR is independently mergeable and deployable, stacked in order. -Reviewed: adversarial verify pass (3 independent critics) applied — see "Verified constraints" below. - -## Overview - -Build the `generic_forms` engine (Form / FormQuestion / FormAnswer, freeze-on-answer, JSON answers with versioned envelope), expose it over GraphQL, wire grants as the first consumer (8 soft questions move from hardcoded `Grant` columns to dynamic form answers), surface answers in grant admin + export, and render the form dynamically on the frontend. - -## Resolved since spec (verified in codebase) - -- `react-hook-form` has **zero** usages despite being in package.json; every form (incl. the modern `invitation-letter-form.tsx`) uses `react-use-form-state`. New `DynamicForm` uses `react-use-form-state`. (Spec §3/§8 corrected.) -- Grant's social columns (`website`, `twitter_handle`, …) are **already dead** — not in the GraphQL `Grant` type, not written by the form (socials go through `Participant` via `PublicProfileCard`). They do NOT become form questions. Soft-question set is exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Spec decision #5 corrected.) -- `send_grant`/`update_grant` are `@transaction.atomic` ([api/grants/mutations.py:226,297](../../backend/api/grants/mutations.py)) — FormAnswer persistence slots into the existing transaction. -- `BaseGrantInput.validate()` (mutations.py:74-111) **mixes** structured-field checks (`full_name`, `grant_type`, departure fields — these STAY) with soft-field checks (max lengths why:1000, python_usage:700, been_to_other_events:500, community_contribution:900, notes:350; required: why, python_usage, been_to_other_events). Only the soft-field portion is superseded by `validate_answers` — structured-field validation must remain untouched. -- Of the 8 soft columns, exactly **4** lack `blank=True` today: `why`, `python_usage`, `been_to_other_events`, `occupation`. The other 4 (`age_group`, `community_contribution`, `gender`, `notes`) are already `blank=True`. All 8 are NOT NULL at the DB level (`blank=True` is Python-only) — `None` must never reach `Grant.objects.create`. -- django-import-export is **3.3.9**; dynamic per-export fields are supported: `Resource.__init__` deep-copies `self.fields` (sanctioned mutation point), and `GrantAdmin.get_export_resource_kwargs(request, ...)` passes context into `GrantResource.__init__`. Extra instance fields auto-append to export order. -- Conference GraphQL pattern to mirror: `deadline(self, info, type: str)` at [api/conferences/types.py:196](../../backend/api/conferences/types.py#L196). Enum pattern: `strawberry.enum(Model.TextChoices)`. -- Tests: model tests in `generic_forms/tests/`, API tests in `api/generic_forms/tests/` + `api/grants/tests/`; `graphql_client` fixture, factory_boy, `pytest.mark.django_db`. -- No read-only-JSON admin precedent exists — the answers display in GrantAdmin is net-new (simple `format_html` list, no new deps). - -## Verified constraints (from the adversarial review — these shape the tasks) - -1. **Dotted `answers.` error paths are impossible.** `BaseErrorType.add_error` getattr-traverses statically-typed error classes (api/types.py:33-74); dynamic keys raise `AttributeError`, and strawberry cannot serialize dynamic field names regardless. The in-repo dotted precedent (`materials.0.url`) lives in **api/submissions** (not visa) and works only because `materials: list[ProposalMaterialErrors]` is statically declared. **Decision (resolved, not a risk): `answers_errors: JSON` field on `_GrantErrors`, set by direct assignment.** Spec §5/§8/§11 updated. Frontend consumes `answersErrors` only. -2. **PR3 must survive the exact PR5 payload.** An answers-only submission (all 8 soft fields omitted) must pass: (a) legacy soft-field required/max-length checks run ONLY on the legacy path (answers not provided); (b) soft input `None` values coalesce to `""` before `Grant.objects.create` / the update setattr loop (DB columns are NOT NULL). A named PR3 test sends answers and omits all 8 soft fields. -3. **Frontend codegen needs a deployed backend schema.** `codegen.yml` fetches the schema from a live endpoint; PR CI (`frontend-lint.yml`) codegens against the staging backend (pastaporto), which deploys only via manual `workflow_dispatch`. **PR5 therefore build-depends on PR3 being deployed to staging**, not merely merged. Release step added before PR5. (Optional improvement, needs approval per spec boundaries — CI change: check in a schema snapshot via `strawberry export-schema` and point codegen at the file.) -4. **Deadline-closed behavior stays as-is** (`non_field_errors: "The grants form is not open!"`). No `FormNotAvailable` union member — changing the response shape breaks the deployed frontend. Spec §5 amended accordingly. `answers` with no GRANT form configured → clear field error. -5. **Production data dependency:** the GRANT form must exist (with the 8 questions) in production admin BEFORE PR5 deploys, or the live form loses its soft questions. Seeding command was explicitly cut from scope → this is a manual ops step in Checkpoint 5, on both staging and production. Frontend must also handle `form == null` by blocking submission with a "form not available" state (never submit without answers). -6. **Legacy-field removal follow-up must be two PRs**, not one: (1) frontend-only — strip legacy `GrantErrors` validation selections (submit-grant.graphql:12-34, pages/grants/edit/update-grant.graphql:25-52) and legacy soft-field selections (my-grant.graphql, update-grant.graphql) — deployable against the unchanged backend; (2) after deploy + soak (stale browser tabs still send old payloads), backend-only — remove the legacy input fields. PR5 already stops *sending* soft fields; it also strips whatever legacy selections it can without breaking its own build. - -## Architecture decisions - -- **Stacked-PR back-compat rule:** every PR leaves `main` deployable (backend deploys before frontend, per deploy.yml ordering). PR3 is strictly additive on the wire: soft fields optional, `answers` optional, legacy shape untouched. -- **Answers storage:** versioned envelope `{"version": 1, "answers": {"": value}}`; GraphQL wire format is the flat map (`strawberry.scalars.JSON`). -- **Question ids as answer keys:** `FormQuestion.pk` stringified; frontend treats them as opaque. -- **Prefill regression accepted and specced** (spec §2 out-of-scope): dateBirth→ageGroup and user.gender prefills drop. -- **Mid-cycle cutover caveat:** grants submitted pre-PR5 (legacy path) have soft answers in columns, not FormAnswer — post-cutover their edit view shows empty dynamic questions. Mitigation: deploy the cutover before grants open for the next conference (ops note in Checkpoint 5); a data backfill is explicitly out of scope. - -## Dependency graph - -``` -PR1 generic_forms app (models + freeze + validate_answers + admin) - ├── PR2 GraphQL query side (Conference.form(purpose)) - └── PR3 grants backend (Grant.form_answer, mutations, Grant.formAnswers) - ├── PR4 grant admin display + export (needs PR3 merged) - └── PR5 frontend DynamicForm + grant form (needs PR2 + PR3 DEPLOYED to staging for codegen/CI) -``` - -Linear stack order: PR1 → PR2 → PR3 → PR4 → PR5. PR4 can start once PR3 merges; PR5 once PR3 reaches staging. - ---- - -## PR1 — `generic_forms` app core (backend only, no consumers) - -Suggested branch: `generic-forms/01-app` - -### Task 1.1: App skeleton + models + migration - -**Description:** Create the `generic_forms` Django app with `Form`, `FormQuestion`, `FormAnswer` models per spec §4 (plain `CharField`/`TextField`, English only), DB constraints, and initial migration. Register in `INSTALLED_APPS` (dotted AppConfig path, `default_auto_field = BigAutoField` like `visa/apps.py`). - -**Acceptance criteria:** -- [ ] Models match spec §4: `Form(conference, purpose, name)`, `FormQuestion(form, label, description, question_type, options, required, max_length, order, active)`, `FormAnswer(form PROTECT, user, answers JSON)`. -- [ ] Constraints enforced at DB level: unique `(form, user)` on FormAnswer; at most one form per `(conference, purpose)` when purpose != `generic` (conditional UniqueConstraint). -- [ ] Migration is plain `makemigrations` output; applies cleanly. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; `uv run python manage.py makemigrations --check --dry-run` clean afterward. - -**Dependencies:** None. -**Files:** `backend/generic_forms/{__init__,apps,models}.py`, `backend/generic_forms/migrations/0001_initial.py`, `backend/pycon/settings/base.py`, `backend/generic_forms/tests/{__init__,factories,test_models}.py` -**Scope:** M - -### Task 1.2: Freeze-on-answer enforcement - -**Description:** Once `form.answers.exists()`: changing `question_type`/`options`/`required` on a `FormQuestion`, or deleting it, raises `ValidationError`; `label`/`description`/`order`/`active` stay editable. Enforced in the model (`clean()` + `save()` guard + `delete()` override). - -**Acceptance criteria:** -- [ ] Semantic-field change on an answered form raises; same change on an unanswered form succeeds. -- [ ] Delete blocked on answered form; `active=False` allowed. -- [ ] Label/description/order edits always allowed. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_models.py -l` green. - -**Dependencies:** 1.1. -**Files:** `backend/generic_forms/models.py`, `backend/generic_forms/tests/test_models.py` -**Scope:** S - -### Task 1.3: `validate_answers` service + envelope helpers - -**Description:** `validate_answers(form, answers: dict) -> dict[str, list[str]]` per spec §4 (unknown/inactive ids, required, per-type checks, option membership incl. every multi_select item, `URLValidator`, `max_length`), plus `wrap_answers` / `unwrap_answers` envelope helpers dispatching on `version`. - -**Acceptance criteria:** -- [ ] Every question type has accept + reject cases covered by tests (spec §9 list). -- [ ] Valid input returns `{}`; errors keyed by question id (this dict is exactly what `answers_errors` carries on the wire in PR3). -- [ ] Envelope round-trip: `unwrap(wrap(x)) == x`; unwrap raises on unknown version. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_services.py -l` green. - -**Dependencies:** 1.1. -**Files:** `backend/generic_forms/services.py`, `backend/generic_forms/tests/test_services.py` -**Scope:** M - -### Task 1.4: Django admin for form authoring - -**Description:** `FormAdmin` with `FormQuestionInline` (TabularInline, mirror `SponsorLevelBenefitInline` simplicity; ordered by `order`), raw JSON widget for `options` (decision #7). Freeze rule surfaces as model validation errors in the inline (deviation applied during build: inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); inline deletion blocked once answered; `Form.conference`/`purpose` readonly once answered. `FormAnswerAdmin` fully read-only (no add/change/delete — deleting answers would unfreeze questions and destroy submissions). - -**Acceptance criteria:** -- [ ] Organizer can create a form + questions of every type entirely in admin (success criterion 1). -- [ ] Inline shows semantic fields readonly once the form has answers. -- [ ] FormAnswer visible but not editable in admin. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; manual: create form with all 6 question types in local admin. - -**Dependencies:** 1.2. -**Files:** `backend/generic_forms/admin.py`, `backend/generic_forms/tests/test_admin.py` -**Scope:** S - -### ▣ CHECKPOINT 1 (end of PR1) -- [ ] `pytest generic_forms`, full `pytest`, `ruff check`, `ruff format --check`, `mypy .` all green. -- [ ] PR1 opened; human review before stacking further. - ---- - -## PR2 — GraphQL query side - -Suggested branch: `generic-forms/02-graphql-query` (stacked on PR1) - -### Task 2.1: Form types + `Conference.form(purpose)` field - -**Description:** `api/generic_forms/types.py`: `FormType`, `FormQuestionType` (id, label, description, questionType, required, maxLength, options as `list[FormQuestionOption(id, label)]`), `FormPurpose = strawberry.enum(Form.Purpose)`, `QuestionType = strawberry.enum(FormQuestion.QuestionType)`. Add `form(self, info, purpose: FormPurpose) -> FormType | None` to the Conference type, mirroring `deadline()`. Questions resolver returns active-only, ordered by `order`. - -**Acceptance criteria:** -- [ ] Query in spec §5 works verbatim. -- [ ] Returns `null` when no form configured; inactive questions excluded; order respected. - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/generic_forms -l` green; `ruff`/`mypy` clean. - -**Dependencies:** PR1. -**Files:** `backend/api/generic_forms/{__init__,types}.py`, `backend/api/conferences/types.py`, `backend/api/generic_forms/tests/{__init__,test_form_query}.py` -**Scope:** S - -### ▣ CHECKPOINT 2 (end of PR2) -- [ ] Full backend suite + lint + types green. GraphQL schema diff reviewed (additive only). PR2 opened. - ---- - -## PR3 — grants backend integration - -Suggested branch: `generic-forms/03-grants-backend` (stacked on PR2) - -### Task 3.1: `Grant.form_answer` link + soft-column loosening - -**Description:** Add `Grant.form_answer = OneToOneField(generic_forms.FormAnswer, null=True, blank=True, SET_NULL)`. Loosen the **4** currently-required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`) to `blank=True` (the other 4 already are). One migration, no data changes. Note: columns remain NOT NULL — the mutation layer must never pass `None` (handled in 3.2). - -**Acceptance criteria:** -- [ ] Migration applies; no other schema changes; historical rows untouched. -- [ ] Existing grants test suite green. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants api/grants -l` green. - -**Dependencies:** PR1. -**Files:** `backend/grants/models.py`, `backend/grants/migrations/00XX_*.py` -**Scope:** S - -### Task 3.2: Mutations accept `answers` (with tests, TDD) - -**Description:** `SendGrantInput`/`UpdateGrantInput`: the 8 soft fields become optional; new optional `answers: JSON`. Validation split: -- Legacy soft-field checks (required + max-length subset of `BaseGrantInput.validate`) run **only** when the legacy path is used (`answers` not provided). Structured-field validation (`full_name`, `grant_type`, departure fields, deadline gating) is **unchanged on both paths**. -- Answers path: reject if no GRANT form configured; else `validate_answers`; failures go into new `answers_errors: JSON` field on `_GrantErrors` by direct assignment (NOT `add_error` — dynamic keys can't traverse the typed class; see Verified constraint 1). -Mutation body: inside the existing `@transaction.atomic`, wrap answers into the envelope, `update_or_create` the FormAnswer, link `grant.form_answer`. Soft input `None` values coalesce to `""` before `Grant.objects.create`; `update_grant`'s `asdict(input)` setattr loop skips `answers` and never writes `None` into soft columns. Tests land in this task (failing-first): answers happy path, invalid answers → `answersErrors` + atomic rollback (no Grant, no FormAnswer), **answers-only payload omitting all 8 soft fields end-to-end (the exact PR5 payload)**, legacy-shape regression (today's payload byte-identical behavior), update-no-duplicate (unique constraint), answers-with-no-form rejected, deadline-closed unchanged, structured-field validation unchanged. - -**Acceptance criteria:** -- [ ] All paths above covered by tests in `api/grants/tests/`; whole grants suite green. -- [ ] Answers-only payload (no soft fields) succeeds — named test. -- [ ] Legacy payload behavior unchanged — named test. -- [ ] No `None` ever written to a NOT NULL soft column (create or update path). - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants grants generic_forms -l` green. - -**Dependencies:** 3.1. -**Files:** `backend/api/grants/mutations.py`, `backend/api/grants/tests/test_send_grant.py`, `backend/api/grants/tests/test_update_grant.py` -**Scope:** M - -### Task 3.3: Expose `Grant.formAnswers` (read side) - -**Description:** `formAnswers: JSON | None` on the `Grant` GraphQL type (api/grants/types.py) returning the unwrapped flat map from the linked FormAnswer, `None` when absent. Used by the edit-flow prefill in PR5. Own query test (via `me.grant`). - -**Acceptance criteria:** -- [ ] `me.grant.formAnswers` returns the flat map for a grant with FormAnswer; `null` for a legacy grant. - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants -l` green. - -**Dependencies:** 3.2. -**Files:** `backend/api/grants/types.py`, `backend/api/grants/tests/test_grant_type.py` (or existing query test file) -**Scope:** XS - -### ▣ CHECKPOINT 3 (end of PR3) -- [ ] Full suite + lint + mypy green. Schema diff additive. -- [ ] Back-compat verified: legacy payload tests green (old frontend deployable against this backend); answers-only payload test green (new frontend's contract already proven). -- [ ] PR3 opened — **key review gate: back-compat story**. -- [ ] After merge: **deploy to staging (pastaporto) via `workflow_dispatch`** — PR5's CI codegen needs this schema live. - ---- - -## PR4 — grant admin display + export - -Suggested branch: `generic-forms/04-admin` (stacked on PR3; can start once PR3 merges) - -### Task 4.1: Read-only answers display in GrantAdmin - -**Description:** New readonly pseudo-field on `GrantAdmin` (in "The Grant" fieldset) rendering the linked FormAnswer as a question-label → answer list via `format_html` (option ids resolved to labels; multi_select joined). Empty state for historical grants. `select_related`/prefetch on the admin queryset (no N+1). - -**Acceptance criteria:** -- [ ] Grant with FormAnswer shows Q/A pairs readonly; grant without shows an empty note; changelist/change view query counts stay flat. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green; manual admin check. - -**Dependencies:** PR3. -**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` -**Scope:** S - -### Task 4.2: Dynamic export columns - -**Description:** `GrantResource.__init__` accepts export context via `GrantAdmin.get_export_resource_kwargs` (import-export 3.3.9 sanctioned path, verified incl. the export-form preview instantiating with the same kwargs), resolves the conference's GRANT form, appends one `Field` per question (column name = question label, `dehydrate_method` reading the FormAnswer). Historical grants export empty cells; legacy soft columns stay in `EXPORT_GRANTS_FIELDS`. - -**Acceptance criteria:** -- [ ] Export of grants with FormAnswers yields one column per question, values resolved (labels for options). -- [ ] Export of a historical conference (no form/answers) unchanged vs today. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green (resource-level tests). - -**Dependencies:** 4.1 (same files). -**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` -**Scope:** M - -### ▣ CHECKPOINT 4 (end of PR4) -- [ ] Full suite + lint + mypy green. Manual: export CSV from local admin with a seeded form. PR4 opened. - ---- - -## PR5 — frontend DynamicForm + grant form integration - -Suggested branch: `generic-forms/05-frontend` (stacked on PR3; **prerequisite: PR3 deployed to staging** so `frontend-lint` codegen sees the new schema) - -### Task 5.1: `DynamicForm` component + fragment - -**Description:** `frontend/src/components/dynamic-form/`: `form.graphql` fragment (form + questions incl. options), `pnpm codegen`, and `index.tsx` rendering each question by `questionType` via styleguide primitives inside `InputWrapper` (mirror `invitation-letter-form.tsx`): text→`Input`, textarea→`Textarea` (+maxLength), select→`Select`, multi_select→`Checkbox` group, boolean→`Checkbox`, url→`Input`. State via the parent's `react-use-form-state` (answers keyed by question id); errors prop consumes the `answersErrors` map (`question_id → string[]`). - -**Acceptance criteria:** -- [ ] Renders all 6 question types from a fragment-typed prop; required marking + maxLength client-side; per-question errors render under fields. -- [ ] No hand edits to generated files. - -**Verification:** `cd frontend && pnpm codegen && pnpm test && pnpm build` green (component test for render-by-type). - -**Dependencies:** PR2 + PR3 deployed to staging (codegen). -**Files:** `frontend/src/components/dynamic-form/{index.tsx,form.graphql,dynamic-form.test.tsx}` (+ regenerated `src/types.tsx`) -**Scope:** M - -### Task 5.2: Grant form integration — new submission flow - -**Description:** `grant-form/index.tsx`: fetch `conference.form(purpose: GRANT)`; replace the 8 hardcoded soft inputs with `DynamicForm`; build the `answers` map on submit and **stop sending the 8 legacy input fields**; map `answersErrors` to the component. **Null-form guard:** if `form` is `null`, block submission and show a "form not available" state — never submit without answers (Verified constraint 5). Strip the legacy `GrantErrors` validation selections for the 8 soft fields from `submit-grant.graphql`. Structured fields (fullName, nationality, grantType, travel/visa/accommodation, PublicProfileCard, privacy checkbox) untouched. Prune dead `options.ts` constants (`GENDER_OPTIONS`, `AGE_GROUPS_OPTIONS`, `OCCUPATION_OPTIONS`) only if nothing else imports them; `GRANT_TYPE_OPTIONS` stays. Accepted regression (specced): dateBirth/gender prefills drop. - -**Acceptance criteria:** -- [ ] New submission works E2E against local backend with a seeded form (success criterion 6: add question in admin → appears on page, no code change). -- [ ] `form == null` → submission blocked with visible message. -- [ ] Per-question server errors display under the right inputs; no legacy soft fields in the mutation payload. - -**Verification:** `cd frontend && pnpm test && pnpm build`; manual: docker-compose, create form in admin, submit a grant. - -**Dependencies:** 5.1. -**Files:** `frontend/src/components/grant-form/index.tsx`, `frontend/src/components/grant-form/submit-grant.graphql`, `frontend/src/components/grant-form/options.ts` -**Scope:** M - -### Task 5.3: Grant form integration — edit flow - -**Description:** Edit-flow prefill from `me.grant.formAnswers`: add `formAnswers` to `pages/grants/edit/my-grant.graphql`, feed into `DynamicForm` initial state; update `pages/grants/edit/update-grant.graphql` (strip legacy soft-field + validation selections, keep structured ones); `pages/grants/edit/index.tsx` passes the form + answers through. Legacy grants (`formAnswers == null`) show empty dynamic questions — accepted mid-cycle caveat (plan decision; cutover deploys before grants open). - -**Acceptance criteria:** -- [ ] Edit flow prefills dynamic answers and saves changes (update path, no duplicate FormAnswer). -- [ ] `pnpm build` green; edit page documents carry no legacy soft-field selections. - -**Verification:** `cd frontend && pnpm test && pnpm build`; manual: edit a grant submitted via the new flow. - -**Dependencies:** 5.2. -**Files:** `frontend/src/pages/grants/edit/{index.tsx,my-grant.graphql,update-grant.graphql}` -**Scope:** S - -### ▣ CHECKPOINT 5 — FINAL -- [ ] All spec §11 success criteria pass (walk the list one by one). -- [ ] Full backend suite, `ruff`, `mypy`, `pnpm test`, `pnpm build` green. -- [ ] Manual E2E on docker-compose: author form → submit grant → edit grant → view in admin → export CSV. -- [ ] **Ops before merging PR5:** GRANT form with the 8 current questions created and verified in **staging AND production** admin (manual — seeding command was cut from scope). Cutover timed **before grants open** for the next conference (pre-existing legacy applications would show empty dynamic questions in edit). -- [ ] Follow-up ticketed as **two** PRs (not in stack): (1) frontend-only — remove remaining legacy `GrantErrors`/`Grant` selections; (2) after deploy + soak, backend-only — remove legacy soft input fields. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| PR3 rejects/500s on the future PR5 payload | High | Verified constraint 2 baked into T3.2: conditional legacy validation, None→"" coalescing, named answers-only test | -| PR5 CI codegen can't see PR3 schema | Med | Explicit staging deploy step in Checkpoint 3; optional schema-snapshot improvement (needs approval — CI change) | -| Production GRANT form missing at PR5 deploy → silent soft-answer loss | High | Null-form guard blocks submission (T5.2); manual ops step in Checkpoint 5 for staging + production | -| Mid-cycle cutover: legacy grants' edit view shows empty questions | Med | Deploy before grants open (Checkpoint 5 ops note); backfill explicitly out of scope | -| Legacy-field removal breaks live clients | Med | Follow-up split into frontend-first + soak + backend PRs (Verified constraint 6) | -| Export preview instantiates resource with same kwargs | Low | Known from source read; resource tests cover it | -| `useFormState` dynamic keys awkward for answers record | Low | Single `answers` object in state; component test proves it before integration | - -## Parallelization - -- PR1 tasks sequential (same files). PR2 once PR1 models stable. -- After PR3 **merges**: PR4 can start. After PR3 **reaches staging**: PR5 can start. PR4 ∥ PR5. - -## Open questions - -None. All decisions resolved (error wire format committed: `answersErrors`; deadline behavior unchanged; ops steps explicit). diff --git a/tasks/generic-forms/todo.md b/tasks/generic-forms/todo.md deleted file mode 100644 index f48a6eb7f6..0000000000 --- a/tasks/generic-forms/todo.md +++ /dev/null @@ -1,32 +0,0 @@ -# TODO: Generic Form System - -Spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Plan: [plan.md](plan.md) -Stack: PR1 → PR2 → PR3 → (PR4 ∥ PR5) · PR5 needs PR3 **deployed to staging** (codegen), not just merged. - -## PR1 — `generic_forms` app core (`generic-forms/01-app`) — **PR #4705** -- [x] **T1.1** App skeleton + `Form`/`FormQuestion`/`FormAnswer` + DB constraints + migration + INSTALLED_APPS. (b885f5d0e) -- [x] **T1.2** Freeze-on-answer in model: type/options/required/form + delete blocked once answered (pre_delete signal); label/order/active free. (3dfe2ee76) -- [x] **T1.3** `validate_answers()` + envelope `wrap/unwrap`. (43bb587d5) -- [x] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze via model validation errors — readonly deviation documented in plan), read-only FormAnswerAdmin incl. delete block. (f9c2a273d) -- [x] **▣ CHECKPOINT 1** — 48 app tests, full suite 1191 green, ruff clean; adversarial review (3 lenses) applied (455525748); PR #4705 open. **Human review pending. Manual admin eyeball pending.** - -## PR2 — GraphQL query (`generic-forms/02-graphql-query`) -- [ ] **T2.1** `api/generic_forms/types.py` (FormType, FormQuestionType, enums, options) + `Conference.form(purpose)` (mirror `deadline()`); active-only ordered; null when unconfigured. Verify: `pytest api/generic_forms`. -- [ ] **▣ CHECKPOINT 2** — suite green; schema diff additive; PR2 opened. - -## PR3 — grants backend (`generic-forms/03-grants-backend`) -- [ ] **T3.1** `Grant.form_answer` OneToOne (SET_NULL) + `blank=True` on the 4 required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`); one migration. Columns stay NOT NULL — mutations must never pass `None`. -- [ ] **T3.2** Mutations + tests (TDD): 8 soft fields optional + optional `answers: JSON`; legacy soft-field checks run only when `answers` absent (structured-field validation unchanged on both paths); errors via `answers_errors: JSON` **direct assignment** (dotted paths impossible — verified); FormAnswer `update_or_create` in existing transaction; `None`→`""` coalescing on create; setattr skip-list on update. Named tests: **answers-only payload (exact PR5 shape)**, legacy-shape regression, invalid→atomic rollback, no-form-configured rejected, deadline-closed unchanged, update-no-duplicate. -- [ ] **T3.3** `Grant.formAnswers: JSON|null` on GraphQL type + query test (`me.grant.formAnswers`). -- [ ] **▣ CHECKPOINT 3** — suite green; back-compat verified both directions; PR3 opened; **after merge: deploy to staging (workflow_dispatch) for PR5 codegen**. - -## PR4 — admin display + export (`generic-forms/04-admin`) -- [ ] **T4.1** GrantAdmin readonly Q/A display (`format_html`), empty state, no N+1. Verify: `pytest grants/tests/test_admin.py` + manual. -- [ ] **T4.2** `GrantResource` dynamic columns via `get_export_resource_kwargs` → `__init__` fields append (3.3.9 verified path, incl. export-form preview); historical export unchanged. Verify: resource tests. -- [ ] **▣ CHECKPOINT 4** — suite green; manual CSV export; PR4 opened. - -## PR5 — frontend (`generic-forms/05-frontend`) — start only after PR3 on staging -- [ ] **T5.1** `dynamic-form/` component + fragment + codegen; 6 types via styleguide + InputWrapper (mirror invitation-letter-form); errors from `answersErrors` map; component test. Verify: `pnpm codegen && pnpm test && pnpm build`. -- [ ] **T5.2** New-submission integration: fetch `form(GRANT)`, swap 8 hardcoded inputs for DynamicForm, `answers` in payload (drop legacy 8), **null-form guard blocks submission**, strip legacy validation selections from submit-grant.graphql, prune dead options.ts constants. Verify: pnpm test/build + manual submit. -- [ ] **T5.3** Edit flow: `formAnswers` into my-grant.graphql, prefill DynamicForm, strip legacy selections from edit documents. Verify: pnpm build + manual edit. -- [ ] **▣ CHECKPOINT 5 — FINAL** — spec §11 walked one-by-one; manual E2E (author → submit → edit → admin → export); **ops: GRANT form created in staging + production admin BEFORE merge; cutover before grants open**; follow-up ticketed as TWO PRs (frontend strip → soak → backend input removal). From a787d85e8504a84505e27a8b0063337fe9afb612 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 7 Aug 2026 04:38:33 +0200 Subject: [PATCH 13/13] Remove spec and task docs from the PR --- specs/generic-form-system.md | 303 --------------------------------- tasks/generic-forms/plan.md | 316 ----------------------------------- tasks/generic-forms/todo.md | 32 ---- 3 files changed, 651 deletions(-) delete mode 100644 specs/generic-form-system.md delete mode 100644 tasks/generic-forms/plan.md delete mode 100644 tasks/generic-forms/todo.md diff --git a/specs/generic-form-system.md b/specs/generic-form-system.md deleted file mode 100644 index da980eba3e..0000000000 --- a/specs/generic-form-system.md +++ /dev/null @@ -1,303 +0,0 @@ -# Spec: Generic Form System - -Status: Approved — ready for planning -Source: Notion draft "Generic Form system" (exported HTML in repo root) + clarifying Q&A -Author: generated via spec-driven-development - ---- - -## 1. Objective - -Build a generic, per-conference configurable form system so organizers can change the questions asked in recurring flows (grants, CFP, visa, feedback) **without backend or frontend code changes**. Today every question is a hardcoded model column (`Grant`, `Submission`) or an external Google Form; changing questions for a new conference edition requires coordinated BE + FE work and migrations. - -**First consumer (this spec's scope): the grant application form.** The engine is built generically; grants is the first flow wired to it. CFP, visa, and feedback forms are explicitly future slices. - -**Target users:** -- *Organizers* — author/edit form questions per conference in Django admin. -- *Attendees/applicants* — fill forms on the Next.js frontend. -- *Maintainers* — stop writing migrations + form components for every question change. - -**Success looks like:** an organizer can add, reword, reorder, or deactivate a grant-form question for the next conference entirely from Django admin, and the frontend renders and validates it with zero code changes. - -### Decisions already made (via Q&A) - -1. **MVP integration target: grants** (biggest pain; `Grant` has ~20 hardcoded answer columns). -2. **Data model: hybrid** — `Form`/`FormQuestion` as normal models (admin-authorable), answers stored as a single `FormAnswer` row per submission with a `JSONField` mapping `question_id → value`. No per-question answer rows. -3. **Versioning: freeze-on-answer** — a question's semantic fields (type, options, required) become immutable once any answer exists for its form. Changes happen by deactivating questions and adding new ones (or cloning the form for a new conference). No snapshot or version-row machinery. -4. **Authoring UI: Django admin** — inline `FormQuestion` editing under `Form`. No custom-admin/Astro builder in this slice. -5. **Load-bearing grant fields stay as `Grant` columns** (confirmed). Fields that drive business logic — `grant_type` (reimbursement categories), `departure_country`/`nationality` (`country_type` derivation, visa), `departure_city`, `needs_funds_for_travel`, `need_visa`, `need_accommodation` — remain structured columns on `Grant`, as do `full_name`/`name`. The *soft* questions moving into the generic form are exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Corrected during planning: socials/website do NOT move — the grant form's social inputs are `participant_*` fields handled via `PublicProfileCard`/`Participant` upsert, not Grant columns; Grant's own social columns are already unused by the current flow.) This avoids a question→field mapping layer in the MVP. -6. **English only** — no multi-lingual labels/options (confirmed). -7. **Options-as-JSON admin UX**: raw JSON widget is acceptable — no custom widget (confirmed). -8. **Grant admin export includes dynamic answers in this slice** (confirmed). The existing `GrantResource` (django-import-export, `grants/admin.py`) exports several soft-question columns today; those move to dynamic-answer columns — one column per question of the conference's grant form (the export is already single-conference via `before_export`). -9. **`purpose` enum values for cfp/visa/feedback are added when those slices land**, not preemptively (confirmed). - -### Assumptions I'm making (correct before approval if wrong) - -1. **No data migration of historical grants.** Old `Grant` columns stay populated and readable for past conferences; new conferences write soft answers to `FormAnswer` only. Legacy columns become nullable/blank-able but are **not dropped** in this slice. -2. **One `FormAnswer` per (form, user).** Matches the existing one-grant-per-user-per-conference constraint. Multi-response generic forms (e.g. anonymous feedback) are future work. -3. **Question labels/descriptions are editable even after answers exist** (typo fixes); only `question_type`, `options`, and `required` freeze. Deletion is blocked once answered — deactivate instead. -4. **New Django app named `generic_forms`** (avoids collision/confusion with `django.forms` and `wagtail.contrib.forms`, which is installed but unused). -5. **Select options live in a `JSONField` on `FormQuestion`** (list of `{id, label}`), not a third model — Django admin can't nest inlines two levels deep, and options-as-JSON keeps authoring on one page. -6. **No file-upload question type in MVP** — it requires extending `files_upload.File.Type`, size limits, and upload permissions. Listed as future work. -7. **No conditional/branching questions in MVP.** - ---- - -## 2. Scope - -### In scope - -- New `generic_forms` Django app: `Form`, `FormQuestion`, `FormAnswer` models + migrations + admin. -- Question types: `text` (single line), `textarea`, `select`, `multi_select`, `boolean`, `url`. -- Server-side answer validation (required, type, option membership, max length, URL format) following the existing `BaseErrorType` pattern. -- GraphQL: query a conference's form by purpose (id, name, ordered active questions with labels/options); mutation to submit/update answers is folded into the existing grant mutations (see §5). -- Grants integration: `sendGrant`/`updateGrant` accept an `answers` input, validate against the conference's grant form, persist a `FormAnswer` linked from `Grant`. -- Frontend: a reusable `DynamicForm` component (styleguide inputs, `react-use-form-state`) rendering questions by type; grant form page renders its soft-question sections dynamically. -- Django admin: grant admin displays the applicant's dynamic answers read-only alongside the structured fields. -- Grant admin export: `GrantResource` gains one column per question of the conference's grant form, populated from the linked `FormAnswer`; legacy soft-question columns stay for historical exports. -- Freeze-on-answer enforcement at the model layer (not just admin). - -### Out of scope (explicitly NOT in this slice) - -- CFP/Submission, visa, and feedback form integrations (engine supports `purpose` values for them, but no product wiring). -- Migrating historical `Grant` answer data into `FormAnswer`; dropping legacy `Grant` columns. -- Custom-admin (Astro) form-builder UI; Wagtail integration. -- File-upload, date, number, or conditional question types. -- Anonymous / multi-response forms. -- Generic "form submitted" confirmation email plumbing (draft's idea — good future win, not now; grants keeps its existing notification path). -- Changes to Pretix, Stripe, or the reimbursement flow. -- Profile-based prefill of dynamic answers (today `ageGroup` prefills from `user.dateBirth` and `gender` from `user.gender`; the generic engine has no per-question semantics, so these prefills are dropped — small accepted UX regression). - ---- - -## 3. Tech stack - -- **Backend:** Django 5.x (existing), PostgreSQL, Strawberry GraphQL. No new Python dependencies expected. -- **Language:** English only — plain `CharField`/`TextField` for labels, descriptions, option labels. No `I18nCharField`/`I18nTextField`. -- **Frontend:** Next.js (existing), TypeScript, Apollo Client with codegen (`pnpm codegen`), `react-use-form-state` (corrected during planning: `react-hook-form` is in package.json but has zero usages in the codebase — every existing form, including the modern invitation-letter form, uses `react-use-form-state`; the new component follows the actual in-repo pattern), `@python-italia/pycon-styleguide` inputs. -- **Admin:** stock Django admin with `TabularInline`/`StackedInline`. - ---- - -## 4. Data model - -```python -# backend/generic_forms/models.py -class Form(TimeStampedModel): - class Purpose(models.TextChoices): - GRANT = "grant", _("Grant") - GENERIC = "generic", _("Generic") # cfp/visa/feedback added in later slices - - conference = models.ForeignKey("conferences.Conference", on_delete=models.CASCADE, - related_name="forms") - purpose = models.CharField(max_length=32, choices=Purpose.choices) - name = models.CharField(max_length=200) - # constraint: at most one form per (conference, purpose) when purpose != GENERIC - - -class FormQuestion(TimeStampedModel): - class QuestionType(models.TextChoices): - TEXT = "text" - TEXTAREA = "textarea" - SELECT = "select" - MULTI_SELECT = "multi_select" - BOOLEAN = "boolean" - URL = "url" - - form = models.ForeignKey(Form, on_delete=models.CASCADE, related_name="questions") - label = models.CharField(max_length=300) - description = models.TextField(blank=True) - question_type = models.CharField(max_length=32, choices=QuestionType.choices) - options = models.JSONField(blank=True, default=list) - # options item shape: {"id": "vegan", "label": "Vegan"} - required = models.BooleanField(default=False) - max_length = models.PositiveIntegerField(null=True, blank=True) - order = models.PositiveIntegerField(default=0) - active = models.BooleanField(default=True) # deactivate instead of delete once answered - - -class FormAnswer(TimeStampedModel): - form = models.ForeignKey(Form, on_delete=models.PROTECT, related_name="answers") - user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) - answers = models.JSONField(default=dict) - # Versioned envelope so the structure can evolve without guessing: - # {"version": 1, "answers": {"": value}} - # version 1 value types by question_type: - # text/textarea/url → str, select → option id (str), - # multi_select → list[str] of option ids, boolean → bool - # Readers dispatch on "version"; writers always write the current version. - # (GraphQL input stays the flat {question_id: value} map — the envelope is - # a storage concern; the mutation wraps it on persist.) - - class Meta: - constraints = [models.UniqueConstraint(fields=["form", "user"], - name="unique_form_answer_per_user")] -``` - -**Grant link:** `Grant.form_answer = models.OneToOneField("generic_forms.FormAnswer", null=True, blank=True, on_delete=models.SET_NULL)`. Soft-question columns on `Grant` become `blank=True` (kept for historical data). - -**Freeze-on-answer rule (model layer):** `FormQuestion.clean()`/`save()` raise if `question_type`, `options`, or `required` change while `self.form.answers.exists()`; deletion is blocked via a `pre_delete` signal (covers queryset deletes too). `label`/`description`/`order`/`active` stay editable. `Form.conference`/`purpose` freeze the same way. In admin the rule surfaces as validation errors on the inline (not readonly fields — inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); the model is the enforcement point. Question `options` are shape-validated at authoring (list of `{id, label}` string pairs, unique ids, required for select types, forbidden otherwise). - -**Answer validation (single source of truth):** a `validate_answers(form, answers: dict) -> dict[str, list[str]]` service in `generic_forms/` used by the GraphQL layer: unknown/inactive question ids rejected, required enforced, per-type checks (option membership incl. every item of multi_select, `URLValidator` for url, `max_length` for text types, bool type check). - ---- - -## 5. API design (GraphQL) - -Follows the newer one-mutation-per-file pattern and the `api/visa/mutations/request_invitation_letter.py` validation style. - -**Query** — extend the existing `Conference` type: - -```graphql -conference(code: "pycon2026") { - form(purpose: GRANT) { # null if no form configured - id - name - questions { # active only, ordered - id - label - description - questionType - required - maxLength - options { id label } - } - } -} -``` - -**Mutations** — no standalone `submitFormAnswers` in this slice. `sendGrant` / `updateGrant` inputs gain an optional `answers: JSON` (map of question id → value). The mutation: -1. Keeps its existing deadline gating unchanged (`non_field_errors: "The grants form is not open!"` via `Conference.is_grants_open`) — no `FormNotAvailable` union member (changing the deadline-closed response shape would break the deployed frontend; decided during planning). -2. If `answers` is provided but the conference has no `GRANT` form, rejects with a clear error. If the form exists, runs `validate_answers`; failures are returned in a dedicated `answersErrors: JSON` field on `GrantErrors` mapping `question_id → [messages]`. (Dotted dynamic paths like `answers.` cannot serialize through the statically-typed error classes — verified during planning; the in-repo dotted-path precedent, `materials.0.url` in `api/submissions`, works only because its container field is statically declared.) -3. Persists `FormAnswer` (create or update), wrapping the input map into the versioned envelope (`{"version": 1, "answers": {...}}`), and links it to the `Grant` in the same transaction. -4. The 8 legacy soft input fields become optional; legacy-shape submissions (soft fields, no `answers`) keep working unchanged until the frontend cutover, then get removed in a post-deploy follow-up. - -**Grant type** — exposes `formAnswers: JSON | null` (the unwrapped flat map) so the frontend edit flow can prefill the dynamic questions. - -Privacy policy acceptance, Slack notification, and email template lookups keep their current grant-specific wiring — unchanged. - ---- - -## 6. Commands - -All backend commands run inside Docker (per CLAUDE.md). - -| Purpose | Command | -|---|---| -| Run backend tests (new app) | `docker exec pycon-backend-1 uv run pytest generic_forms/tests api/generic_forms -l -s -vvv` | -| Grants integration tests | `docker exec pycon-backend-1 uv run pytest api/grants grants -l -s -vvv` | -| Full suite | `docker exec pycon-backend-1 uv run pytest` | -| Make migrations | `docker exec pycon-backend-1 uv run python manage.py makemigrations generic_forms grants` | -| Migrate | `docker exec pycon-backend-1 uv run python manage.py migrate` | -| Lint / format | `docker exec pycon-backend-1 uv run ruff check` / `uv run ruff format` | -| Type check | `docker exec pycon-backend-1 uv run mypy .` | -| Frontend codegen (after schema change) | `cd frontend && pnpm codegen` | -| Frontend tests / build | `cd frontend && pnpm test` / `pnpm build` | - ---- - -## 7. Project structure - -``` -backend/ - generic_forms/ # NEW app - models.py # Form, FormQuestion, FormAnswer - services.py # validate_answers() - admin.py # Form admin + FormQuestion inline (freeze-aware) - migrations/ - tests/ # model + validation tests, factories - api/ - generic_forms/ # NEW: FormType, FormQuestionType (query side) - types.py - grants/mutations.py # extend sendGrant/updateGrant with answers - grants/ - models.py # + form_answer FK; soft columns → blank=True - admin.py # + read-only answers display - pycon/settings/base.py # + generic_forms in INSTALLED_APPS - -frontend/src/ - components/dynamic-form/ # NEW: renders FormQuestion[] via styleguide inputs - index.tsx - form.graphql # fragment for form + questions - components/grant-form/ # integrate DynamicForm for soft questions -``` - ---- - -## 8. Code style - -Backend follows existing conventions — Ruff (lint + format), mypy clean. Mutation validation mirrors the in-repo pattern: - -```python -@strawberry.input -class SendGrantInput: - conference: strawberry.ID - answers: JSON - ... - - def validate(self, conference: Conference, form: Form) -> GrantErrors | None: - errors = GrantErrors() - if answer_errors := validate_answers(form, self.answers): - # dedicated JSON field: {question_id: [messages]} — dynamic keys - # cannot serialize through the statically-typed error fields - errors.answers_errors = answer_errors - return errors.if_has_errors -``` - -Frontend: `react-use-form-state` + `@python-italia/pycon-styleguide` primitives (mirror `invitation-letter-form.tsx`: `InputWrapper` around each field, `MultiplePartsCard` sections); GraphQL documents co-located with components; **never hand-edit generated files** (`src/types.tsx`, `src/generated/`). - ---- - -## 9. Testing strategy - -- **Framework:** pytest + factory-based fixtures, in-app `tests/` dirs (existing convention). Frontend: existing `pnpm test` setup for the `DynamicForm` component's rendering/validation mapping. -- **Model tests** (`generic_forms/tests/`): freeze-on-answer (type/options/required change and delete blocked once an answer exists; label/order/active edits allowed); unique (form, user) constraint; one-form-per-(conference, purpose) constraint. -- **Validation tests:** each question type's accept/reject cases — required missing, wrong value type, unknown question id, inactive question id, non-member option, multi_select with one bad item, invalid URL, over max_length. -- **API tests** (`api/` tests): query returns only active questions in order; `sendGrant` with valid answers creates `Grant` + linked `FormAnswer` atomically; invalid answers return per-question errors in `answersErrors` and persist nothing; answers-with-no-form-configured is rejected; grants-deadline-closed behavior unchanged from today; an answers-only payload omitting all 8 legacy soft fields succeeds end-to-end (this is the exact post-cutover frontend payload). -- **Export test:** `GrantResource` export of a grant with a linked `FormAnswer` produces one column per form question with the answer values (option ids resolved to labels); grants without `FormAnswer` (historical) still export cleanly. -- **Regression:** full existing grants test suite stays green — legacy columns still accepted for old data paths. -- Every slice lands with its tests; `pytest`, `ruff check`, `mypy .` green before any commit. - ---- - -## 10. Boundaries - -### Always do -- Run backend commands via `docker exec pycon-backend-1 ...` (local venv doesn't work). -- Run `pytest` + `ruff check` + `mypy .` (and `pnpm codegen` after schema changes) before committing. -- Enforce freeze-on-answer in the model, not only in admin. -- Validate answers server-side via `validate_answers` — frontend validation is UX only. -- Keep legacy `Grant` columns readable (admin, exports) for historical conferences. - -### Ask first -- Adding any new dependency (backend or frontend). -- Changing which `Grant` fields count as load-bearing (decision #5) — i.e. moving `grant_type`, country, or `need_*` fields into the form. -- Any data migration touching existing `Grant` rows beyond `blank=True` loosening. -- Adding new values to `files_upload.File.Type` (file-upload question type). -- Schema changes to `Submission`, visa, or notification models. -- Dropping or renaming any existing column. - -### Never -- Drop legacy `Grant` answer columns in this slice. -- Hand-edit generated GraphQL types (`frontend/src/types.tsx`, `*.generated.ts`). -- Store answers as per-question rows (decision: JSON) or bypass `validate_answers` in any mutation. -- Commit secrets; weaken rate-limit/permission classes on mutations. -- Delete or skip failing tests to get green. - ---- - -## 11. Success criteria - -1. Organizer creates a `GRANT` form with questions of every supported type in Django admin, reorders and deactivates questions — no code change needed. -2. Once one answer exists, changing a question's type/options/required or deleting it fails with a clear error in both admin and direct model save; label typo fix still succeeds. -3. `conference.form(purpose: GRANT)` returns the ordered active questions; returns `null` when unconfigured. -4. `sendGrant` with valid `answers` creates `Grant` + linked `FormAnswer` in one transaction; a second submit by the same user for the same conference updates rather than duplicates (existing update path). -5. `sendGrant` with an invalid answer (missing required, bad option, invalid URL) returns per-question errors (`answersErrors` map) and writes nothing; an answers-only payload with no legacy soft fields succeeds. -6. Grant form page on the frontend renders the soft-question sections from the API (verify: add a question in admin → it appears on the page after reload, no deploy of new code). -7. Grant admin shows the applicant's dynamic answers read-only next to structured fields. -8. Grant admin export includes a column per form question with the applicant's answers; exports of historical grants (no `FormAnswer`) still work. -9. Full backend test suite, `ruff check`, `mypy .`, frontend `pnpm build` + `pnpm test` all green. - -## 12. Open questions - -None — all resolved into decisions #7–#9. diff --git a/tasks/generic-forms/plan.md b/tasks/generic-forms/plan.md deleted file mode 100644 index add0215ed1..0000000000 --- a/tasks/generic-forms/plan.md +++ /dev/null @@ -1,316 +0,0 @@ -# Implementation Plan: Generic Form System - -Source spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Mode: plan (read-only, no code changed) -Structure: **5 stacked PRs** — each PR is independently mergeable and deployable, stacked in order. -Reviewed: adversarial verify pass (3 independent critics) applied — see "Verified constraints" below. - -## Overview - -Build the `generic_forms` engine (Form / FormQuestion / FormAnswer, freeze-on-answer, JSON answers with versioned envelope), expose it over GraphQL, wire grants as the first consumer (8 soft questions move from hardcoded `Grant` columns to dynamic form answers), surface answers in grant admin + export, and render the form dynamically on the frontend. - -## Resolved since spec (verified in codebase) - -- `react-hook-form` has **zero** usages despite being in package.json; every form (incl. the modern `invitation-letter-form.tsx`) uses `react-use-form-state`. New `DynamicForm` uses `react-use-form-state`. (Spec §3/§8 corrected.) -- Grant's social columns (`website`, `twitter_handle`, …) are **already dead** — not in the GraphQL `Grant` type, not written by the form (socials go through `Participant` via `PublicProfileCard`). They do NOT become form questions. Soft-question set is exactly: `why`, `python_usage`, `been_to_other_events`, `community_contribution`, `age_group`, `occupation`, `gender`, `notes`. (Spec decision #5 corrected.) -- `send_grant`/`update_grant` are `@transaction.atomic` ([api/grants/mutations.py:226,297](../../backend/api/grants/mutations.py)) — FormAnswer persistence slots into the existing transaction. -- `BaseGrantInput.validate()` (mutations.py:74-111) **mixes** structured-field checks (`full_name`, `grant_type`, departure fields — these STAY) with soft-field checks (max lengths why:1000, python_usage:700, been_to_other_events:500, community_contribution:900, notes:350; required: why, python_usage, been_to_other_events). Only the soft-field portion is superseded by `validate_answers` — structured-field validation must remain untouched. -- Of the 8 soft columns, exactly **4** lack `blank=True` today: `why`, `python_usage`, `been_to_other_events`, `occupation`. The other 4 (`age_group`, `community_contribution`, `gender`, `notes`) are already `blank=True`. All 8 are NOT NULL at the DB level (`blank=True` is Python-only) — `None` must never reach `Grant.objects.create`. -- django-import-export is **3.3.9**; dynamic per-export fields are supported: `Resource.__init__` deep-copies `self.fields` (sanctioned mutation point), and `GrantAdmin.get_export_resource_kwargs(request, ...)` passes context into `GrantResource.__init__`. Extra instance fields auto-append to export order. -- Conference GraphQL pattern to mirror: `deadline(self, info, type: str)` at [api/conferences/types.py:196](../../backend/api/conferences/types.py#L196). Enum pattern: `strawberry.enum(Model.TextChoices)`. -- Tests: model tests in `generic_forms/tests/`, API tests in `api/generic_forms/tests/` + `api/grants/tests/`; `graphql_client` fixture, factory_boy, `pytest.mark.django_db`. -- No read-only-JSON admin precedent exists — the answers display in GrantAdmin is net-new (simple `format_html` list, no new deps). - -## Verified constraints (from the adversarial review — these shape the tasks) - -1. **Dotted `answers.` error paths are impossible.** `BaseErrorType.add_error` getattr-traverses statically-typed error classes (api/types.py:33-74); dynamic keys raise `AttributeError`, and strawberry cannot serialize dynamic field names regardless. The in-repo dotted precedent (`materials.0.url`) lives in **api/submissions** (not visa) and works only because `materials: list[ProposalMaterialErrors]` is statically declared. **Decision (resolved, not a risk): `answers_errors: JSON` field on `_GrantErrors`, set by direct assignment.** Spec §5/§8/§11 updated. Frontend consumes `answersErrors` only. -2. **PR3 must survive the exact PR5 payload.** An answers-only submission (all 8 soft fields omitted) must pass: (a) legacy soft-field required/max-length checks run ONLY on the legacy path (answers not provided); (b) soft input `None` values coalesce to `""` before `Grant.objects.create` / the update setattr loop (DB columns are NOT NULL). A named PR3 test sends answers and omits all 8 soft fields. -3. **Frontend codegen needs a deployed backend schema.** `codegen.yml` fetches the schema from a live endpoint; PR CI (`frontend-lint.yml`) codegens against the staging backend (pastaporto), which deploys only via manual `workflow_dispatch`. **PR5 therefore build-depends on PR3 being deployed to staging**, not merely merged. Release step added before PR5. (Optional improvement, needs approval per spec boundaries — CI change: check in a schema snapshot via `strawberry export-schema` and point codegen at the file.) -4. **Deadline-closed behavior stays as-is** (`non_field_errors: "The grants form is not open!"`). No `FormNotAvailable` union member — changing the response shape breaks the deployed frontend. Spec §5 amended accordingly. `answers` with no GRANT form configured → clear field error. -5. **Production data dependency:** the GRANT form must exist (with the 8 questions) in production admin BEFORE PR5 deploys, or the live form loses its soft questions. Seeding command was explicitly cut from scope → this is a manual ops step in Checkpoint 5, on both staging and production. Frontend must also handle `form == null` by blocking submission with a "form not available" state (never submit without answers). -6. **Legacy-field removal follow-up must be two PRs**, not one: (1) frontend-only — strip legacy `GrantErrors` validation selections (submit-grant.graphql:12-34, pages/grants/edit/update-grant.graphql:25-52) and legacy soft-field selections (my-grant.graphql, update-grant.graphql) — deployable against the unchanged backend; (2) after deploy + soak (stale browser tabs still send old payloads), backend-only — remove the legacy input fields. PR5 already stops *sending* soft fields; it also strips whatever legacy selections it can without breaking its own build. - -## Architecture decisions - -- **Stacked-PR back-compat rule:** every PR leaves `main` deployable (backend deploys before frontend, per deploy.yml ordering). PR3 is strictly additive on the wire: soft fields optional, `answers` optional, legacy shape untouched. -- **Answers storage:** versioned envelope `{"version": 1, "answers": {"": value}}`; GraphQL wire format is the flat map (`strawberry.scalars.JSON`). -- **Question ids as answer keys:** `FormQuestion.pk` stringified; frontend treats them as opaque. -- **Prefill regression accepted and specced** (spec §2 out-of-scope): dateBirth→ageGroup and user.gender prefills drop. -- **Mid-cycle cutover caveat:** grants submitted pre-PR5 (legacy path) have soft answers in columns, not FormAnswer — post-cutover their edit view shows empty dynamic questions. Mitigation: deploy the cutover before grants open for the next conference (ops note in Checkpoint 5); a data backfill is explicitly out of scope. - -## Dependency graph - -``` -PR1 generic_forms app (models + freeze + validate_answers + admin) - ├── PR2 GraphQL query side (Conference.form(purpose)) - └── PR3 grants backend (Grant.form_answer, mutations, Grant.formAnswers) - ├── PR4 grant admin display + export (needs PR3 merged) - └── PR5 frontend DynamicForm + grant form (needs PR2 + PR3 DEPLOYED to staging for codegen/CI) -``` - -Linear stack order: PR1 → PR2 → PR3 → PR4 → PR5. PR4 can start once PR3 merges; PR5 once PR3 reaches staging. - ---- - -## PR1 — `generic_forms` app core (backend only, no consumers) - -Suggested branch: `generic-forms/01-app` - -### Task 1.1: App skeleton + models + migration - -**Description:** Create the `generic_forms` Django app with `Form`, `FormQuestion`, `FormAnswer` models per spec §4 (plain `CharField`/`TextField`, English only), DB constraints, and initial migration. Register in `INSTALLED_APPS` (dotted AppConfig path, `default_auto_field = BigAutoField` like `visa/apps.py`). - -**Acceptance criteria:** -- [ ] Models match spec §4: `Form(conference, purpose, name)`, `FormQuestion(form, label, description, question_type, options, required, max_length, order, active)`, `FormAnswer(form PROTECT, user, answers JSON)`. -- [ ] Constraints enforced at DB level: unique `(form, user)` on FormAnswer; at most one form per `(conference, purpose)` when purpose != `generic` (conditional UniqueConstraint). -- [ ] Migration is plain `makemigrations` output; applies cleanly. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; `uv run python manage.py makemigrations --check --dry-run` clean afterward. - -**Dependencies:** None. -**Files:** `backend/generic_forms/{__init__,apps,models}.py`, `backend/generic_forms/migrations/0001_initial.py`, `backend/pycon/settings/base.py`, `backend/generic_forms/tests/{__init__,factories,test_models}.py` -**Scope:** M - -### Task 1.2: Freeze-on-answer enforcement - -**Description:** Once `form.answers.exists()`: changing `question_type`/`options`/`required` on a `FormQuestion`, or deleting it, raises `ValidationError`; `label`/`description`/`order`/`active` stay editable. Enforced in the model (`clean()` + `save()` guard + `delete()` override). - -**Acceptance criteria:** -- [ ] Semantic-field change on an answered form raises; same change on an unanswered form succeeds. -- [ ] Delete blocked on answered form; `active=False` allowed. -- [ ] Label/description/order edits always allowed. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_models.py -l` green. - -**Dependencies:** 1.1. -**Files:** `backend/generic_forms/models.py`, `backend/generic_forms/tests/test_models.py` -**Scope:** S - -### Task 1.3: `validate_answers` service + envelope helpers - -**Description:** `validate_answers(form, answers: dict) -> dict[str, list[str]]` per spec §4 (unknown/inactive ids, required, per-type checks, option membership incl. every multi_select item, `URLValidator`, `max_length`), plus `wrap_answers` / `unwrap_answers` envelope helpers dispatching on `version`. - -**Acceptance criteria:** -- [ ] Every question type has accept + reject cases covered by tests (spec §9 list). -- [ ] Valid input returns `{}`; errors keyed by question id (this dict is exactly what `answers_errors` carries on the wire in PR3). -- [ ] Envelope round-trip: `unwrap(wrap(x)) == x`; unwrap raises on unknown version. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms/tests/test_services.py -l` green. - -**Dependencies:** 1.1. -**Files:** `backend/generic_forms/services.py`, `backend/generic_forms/tests/test_services.py` -**Scope:** M - -### Task 1.4: Django admin for form authoring - -**Description:** `FormAdmin` with `FormQuestionInline` (TabularInline, mirror `SponsorLevelBenefitInline` simplicity; ordered by `order`), raw JSON widget for `options` (decision #7). Freeze rule surfaces as model validation errors in the inline (deviation applied during build: inline-level readonly would also freeze NEW rows, and adding questions to answered forms must stay possible); inline deletion blocked once answered; `Form.conference`/`purpose` readonly once answered. `FormAnswerAdmin` fully read-only (no add/change/delete — deleting answers would unfreeze questions and destroy submissions). - -**Acceptance criteria:** -- [ ] Organizer can create a form + questions of every type entirely in admin (success criterion 1). -- [ ] Inline shows semantic fields readonly once the form has answers. -- [ ] FormAnswer visible but not editable in admin. - -**Verification:** `docker exec pycon-backend-1 uv run pytest generic_forms -l` green; manual: create form with all 6 question types in local admin. - -**Dependencies:** 1.2. -**Files:** `backend/generic_forms/admin.py`, `backend/generic_forms/tests/test_admin.py` -**Scope:** S - -### ▣ CHECKPOINT 1 (end of PR1) -- [ ] `pytest generic_forms`, full `pytest`, `ruff check`, `ruff format --check`, `mypy .` all green. -- [ ] PR1 opened; human review before stacking further. - ---- - -## PR2 — GraphQL query side - -Suggested branch: `generic-forms/02-graphql-query` (stacked on PR1) - -### Task 2.1: Form types + `Conference.form(purpose)` field - -**Description:** `api/generic_forms/types.py`: `FormType`, `FormQuestionType` (id, label, description, questionType, required, maxLength, options as `list[FormQuestionOption(id, label)]`), `FormPurpose = strawberry.enum(Form.Purpose)`, `QuestionType = strawberry.enum(FormQuestion.QuestionType)`. Add `form(self, info, purpose: FormPurpose) -> FormType | None` to the Conference type, mirroring `deadline()`. Questions resolver returns active-only, ordered by `order`. - -**Acceptance criteria:** -- [ ] Query in spec §5 works verbatim. -- [ ] Returns `null` when no form configured; inactive questions excluded; order respected. - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/generic_forms -l` green; `ruff`/`mypy` clean. - -**Dependencies:** PR1. -**Files:** `backend/api/generic_forms/{__init__,types}.py`, `backend/api/conferences/types.py`, `backend/api/generic_forms/tests/{__init__,test_form_query}.py` -**Scope:** S - -### ▣ CHECKPOINT 2 (end of PR2) -- [ ] Full backend suite + lint + types green. GraphQL schema diff reviewed (additive only). PR2 opened. - ---- - -## PR3 — grants backend integration - -Suggested branch: `generic-forms/03-grants-backend` (stacked on PR2) - -### Task 3.1: `Grant.form_answer` link + soft-column loosening - -**Description:** Add `Grant.form_answer = OneToOneField(generic_forms.FormAnswer, null=True, blank=True, SET_NULL)`. Loosen the **4** currently-required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`) to `blank=True` (the other 4 already are). One migration, no data changes. Note: columns remain NOT NULL — the mutation layer must never pass `None` (handled in 3.2). - -**Acceptance criteria:** -- [ ] Migration applies; no other schema changes; historical rows untouched. -- [ ] Existing grants test suite green. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants api/grants -l` green. - -**Dependencies:** PR1. -**Files:** `backend/grants/models.py`, `backend/grants/migrations/00XX_*.py` -**Scope:** S - -### Task 3.2: Mutations accept `answers` (with tests, TDD) - -**Description:** `SendGrantInput`/`UpdateGrantInput`: the 8 soft fields become optional; new optional `answers: JSON`. Validation split: -- Legacy soft-field checks (required + max-length subset of `BaseGrantInput.validate`) run **only** when the legacy path is used (`answers` not provided). Structured-field validation (`full_name`, `grant_type`, departure fields, deadline gating) is **unchanged on both paths**. -- Answers path: reject if no GRANT form configured; else `validate_answers`; failures go into new `answers_errors: JSON` field on `_GrantErrors` by direct assignment (NOT `add_error` — dynamic keys can't traverse the typed class; see Verified constraint 1). -Mutation body: inside the existing `@transaction.atomic`, wrap answers into the envelope, `update_or_create` the FormAnswer, link `grant.form_answer`. Soft input `None` values coalesce to `""` before `Grant.objects.create`; `update_grant`'s `asdict(input)` setattr loop skips `answers` and never writes `None` into soft columns. Tests land in this task (failing-first): answers happy path, invalid answers → `answersErrors` + atomic rollback (no Grant, no FormAnswer), **answers-only payload omitting all 8 soft fields end-to-end (the exact PR5 payload)**, legacy-shape regression (today's payload byte-identical behavior), update-no-duplicate (unique constraint), answers-with-no-form rejected, deadline-closed unchanged, structured-field validation unchanged. - -**Acceptance criteria:** -- [ ] All paths above covered by tests in `api/grants/tests/`; whole grants suite green. -- [ ] Answers-only payload (no soft fields) succeeds — named test. -- [ ] Legacy payload behavior unchanged — named test. -- [ ] No `None` ever written to a NOT NULL soft column (create or update path). - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants grants generic_forms -l` green. - -**Dependencies:** 3.1. -**Files:** `backend/api/grants/mutations.py`, `backend/api/grants/tests/test_send_grant.py`, `backend/api/grants/tests/test_update_grant.py` -**Scope:** M - -### Task 3.3: Expose `Grant.formAnswers` (read side) - -**Description:** `formAnswers: JSON | None` on the `Grant` GraphQL type (api/grants/types.py) returning the unwrapped flat map from the linked FormAnswer, `None` when absent. Used by the edit-flow prefill in PR5. Own query test (via `me.grant`). - -**Acceptance criteria:** -- [ ] `me.grant.formAnswers` returns the flat map for a grant with FormAnswer; `null` for a legacy grant. - -**Verification:** `docker exec pycon-backend-1 uv run pytest api/grants -l` green. - -**Dependencies:** 3.2. -**Files:** `backend/api/grants/types.py`, `backend/api/grants/tests/test_grant_type.py` (or existing query test file) -**Scope:** XS - -### ▣ CHECKPOINT 3 (end of PR3) -- [ ] Full suite + lint + mypy green. Schema diff additive. -- [ ] Back-compat verified: legacy payload tests green (old frontend deployable against this backend); answers-only payload test green (new frontend's contract already proven). -- [ ] PR3 opened — **key review gate: back-compat story**. -- [ ] After merge: **deploy to staging (pastaporto) via `workflow_dispatch`** — PR5's CI codegen needs this schema live. - ---- - -## PR4 — grant admin display + export - -Suggested branch: `generic-forms/04-admin` (stacked on PR3; can start once PR3 merges) - -### Task 4.1: Read-only answers display in GrantAdmin - -**Description:** New readonly pseudo-field on `GrantAdmin` (in "The Grant" fieldset) rendering the linked FormAnswer as a question-label → answer list via `format_html` (option ids resolved to labels; multi_select joined). Empty state for historical grants. `select_related`/prefetch on the admin queryset (no N+1). - -**Acceptance criteria:** -- [ ] Grant with FormAnswer shows Q/A pairs readonly; grant without shows an empty note; changelist/change view query counts stay flat. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green; manual admin check. - -**Dependencies:** PR3. -**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` -**Scope:** S - -### Task 4.2: Dynamic export columns - -**Description:** `GrantResource.__init__` accepts export context via `GrantAdmin.get_export_resource_kwargs` (import-export 3.3.9 sanctioned path, verified incl. the export-form preview instantiating with the same kwargs), resolves the conference's GRANT form, appends one `Field` per question (column name = question label, `dehydrate_method` reading the FormAnswer). Historical grants export empty cells; legacy soft columns stay in `EXPORT_GRANTS_FIELDS`. - -**Acceptance criteria:** -- [ ] Export of grants with FormAnswers yields one column per question, values resolved (labels for options). -- [ ] Export of a historical conference (no form/answers) unchanged vs today. - -**Verification:** `docker exec pycon-backend-1 uv run pytest grants/tests/test_admin.py -l` green (resource-level tests). - -**Dependencies:** 4.1 (same files). -**Files:** `backend/grants/admin.py`, `backend/grants/tests/test_admin.py` -**Scope:** M - -### ▣ CHECKPOINT 4 (end of PR4) -- [ ] Full suite + lint + mypy green. Manual: export CSV from local admin with a seeded form. PR4 opened. - ---- - -## PR5 — frontend DynamicForm + grant form integration - -Suggested branch: `generic-forms/05-frontend` (stacked on PR3; **prerequisite: PR3 deployed to staging** so `frontend-lint` codegen sees the new schema) - -### Task 5.1: `DynamicForm` component + fragment - -**Description:** `frontend/src/components/dynamic-form/`: `form.graphql` fragment (form + questions incl. options), `pnpm codegen`, and `index.tsx` rendering each question by `questionType` via styleguide primitives inside `InputWrapper` (mirror `invitation-letter-form.tsx`): text→`Input`, textarea→`Textarea` (+maxLength), select→`Select`, multi_select→`Checkbox` group, boolean→`Checkbox`, url→`Input`. State via the parent's `react-use-form-state` (answers keyed by question id); errors prop consumes the `answersErrors` map (`question_id → string[]`). - -**Acceptance criteria:** -- [ ] Renders all 6 question types from a fragment-typed prop; required marking + maxLength client-side; per-question errors render under fields. -- [ ] No hand edits to generated files. - -**Verification:** `cd frontend && pnpm codegen && pnpm test && pnpm build` green (component test for render-by-type). - -**Dependencies:** PR2 + PR3 deployed to staging (codegen). -**Files:** `frontend/src/components/dynamic-form/{index.tsx,form.graphql,dynamic-form.test.tsx}` (+ regenerated `src/types.tsx`) -**Scope:** M - -### Task 5.2: Grant form integration — new submission flow - -**Description:** `grant-form/index.tsx`: fetch `conference.form(purpose: GRANT)`; replace the 8 hardcoded soft inputs with `DynamicForm`; build the `answers` map on submit and **stop sending the 8 legacy input fields**; map `answersErrors` to the component. **Null-form guard:** if `form` is `null`, block submission and show a "form not available" state — never submit without answers (Verified constraint 5). Strip the legacy `GrantErrors` validation selections for the 8 soft fields from `submit-grant.graphql`. Structured fields (fullName, nationality, grantType, travel/visa/accommodation, PublicProfileCard, privacy checkbox) untouched. Prune dead `options.ts` constants (`GENDER_OPTIONS`, `AGE_GROUPS_OPTIONS`, `OCCUPATION_OPTIONS`) only if nothing else imports them; `GRANT_TYPE_OPTIONS` stays. Accepted regression (specced): dateBirth/gender prefills drop. - -**Acceptance criteria:** -- [ ] New submission works E2E against local backend with a seeded form (success criterion 6: add question in admin → appears on page, no code change). -- [ ] `form == null` → submission blocked with visible message. -- [ ] Per-question server errors display under the right inputs; no legacy soft fields in the mutation payload. - -**Verification:** `cd frontend && pnpm test && pnpm build`; manual: docker-compose, create form in admin, submit a grant. - -**Dependencies:** 5.1. -**Files:** `frontend/src/components/grant-form/index.tsx`, `frontend/src/components/grant-form/submit-grant.graphql`, `frontend/src/components/grant-form/options.ts` -**Scope:** M - -### Task 5.3: Grant form integration — edit flow - -**Description:** Edit-flow prefill from `me.grant.formAnswers`: add `formAnswers` to `pages/grants/edit/my-grant.graphql`, feed into `DynamicForm` initial state; update `pages/grants/edit/update-grant.graphql` (strip legacy soft-field + validation selections, keep structured ones); `pages/grants/edit/index.tsx` passes the form + answers through. Legacy grants (`formAnswers == null`) show empty dynamic questions — accepted mid-cycle caveat (plan decision; cutover deploys before grants open). - -**Acceptance criteria:** -- [ ] Edit flow prefills dynamic answers and saves changes (update path, no duplicate FormAnswer). -- [ ] `pnpm build` green; edit page documents carry no legacy soft-field selections. - -**Verification:** `cd frontend && pnpm test && pnpm build`; manual: edit a grant submitted via the new flow. - -**Dependencies:** 5.2. -**Files:** `frontend/src/pages/grants/edit/{index.tsx,my-grant.graphql,update-grant.graphql}` -**Scope:** S - -### ▣ CHECKPOINT 5 — FINAL -- [ ] All spec §11 success criteria pass (walk the list one by one). -- [ ] Full backend suite, `ruff`, `mypy`, `pnpm test`, `pnpm build` green. -- [ ] Manual E2E on docker-compose: author form → submit grant → edit grant → view in admin → export CSV. -- [ ] **Ops before merging PR5:** GRANT form with the 8 current questions created and verified in **staging AND production** admin (manual — seeding command was cut from scope). Cutover timed **before grants open** for the next conference (pre-existing legacy applications would show empty dynamic questions in edit). -- [ ] Follow-up ticketed as **two** PRs (not in stack): (1) frontend-only — remove remaining legacy `GrantErrors`/`Grant` selections; (2) after deploy + soak, backend-only — remove legacy soft input fields. - ---- - -## Risks and mitigations - -| Risk | Impact | Mitigation | -|---|---|---| -| PR3 rejects/500s on the future PR5 payload | High | Verified constraint 2 baked into T3.2: conditional legacy validation, None→"" coalescing, named answers-only test | -| PR5 CI codegen can't see PR3 schema | Med | Explicit staging deploy step in Checkpoint 3; optional schema-snapshot improvement (needs approval — CI change) | -| Production GRANT form missing at PR5 deploy → silent soft-answer loss | High | Null-form guard blocks submission (T5.2); manual ops step in Checkpoint 5 for staging + production | -| Mid-cycle cutover: legacy grants' edit view shows empty questions | Med | Deploy before grants open (Checkpoint 5 ops note); backfill explicitly out of scope | -| Legacy-field removal breaks live clients | Med | Follow-up split into frontend-first + soak + backend PRs (Verified constraint 6) | -| Export preview instantiates resource with same kwargs | Low | Known from source read; resource tests cover it | -| `useFormState` dynamic keys awkward for answers record | Low | Single `answers` object in state; component test proves it before integration | - -## Parallelization - -- PR1 tasks sequential (same files). PR2 once PR1 models stable. -- After PR3 **merges**: PR4 can start. After PR3 **reaches staging**: PR5 can start. PR4 ∥ PR5. - -## Open questions - -None. All decisions resolved (error wire format committed: `answersErrors`; deadline behavior unchanged; ops steps explicit). diff --git a/tasks/generic-forms/todo.md b/tasks/generic-forms/todo.md deleted file mode 100644 index 39bb62d554..0000000000 --- a/tasks/generic-forms/todo.md +++ /dev/null @@ -1,32 +0,0 @@ -# TODO: Generic Form System - -Spec: [specs/generic-form-system.md](../../specs/generic-form-system.md) · Plan: [plan.md](plan.md) -Stack: PR1 → PR2 → PR3 → (PR4 ∥ PR5) · PR5 needs PR3 **deployed to staging** (codegen), not just merged. - -## PR1 — `generic_forms` app core (`generic-forms/01-app`) — **PR #4705** -- [x] **T1.1** App skeleton + `Form`/`FormQuestion`/`FormAnswer` + DB constraints + migration + INSTALLED_APPS. (b885f5d0e) -- [x] **T1.2** Freeze-on-answer in model: type/options/required/form + delete blocked once answered (pre_delete signal); label/order/active free. (3dfe2ee76) -- [x] **T1.3** `validate_answers()` + envelope `wrap/unwrap`. (43bb587d5) -- [x] **T1.4** Admin: FormAdmin + FormQuestionInline (freeze via model validation errors — readonly deviation documented in plan), read-only FormAnswerAdmin incl. delete block. (f9c2a273d) -- [x] **▣ CHECKPOINT 1** — 48 app tests, full suite 1191 green, ruff clean; adversarial review (3 lenses) applied (455525748); PR #4705 open. **Human review pending. Manual admin eyeball pending.** - -## PR2 — GraphQL query (`generic-forms/02-graphql-query`) — **PR #4707** -- [x] **T2.1** `api/generic_forms/types.py` (Form, FormQuestion, FormQuestionOption, FormPurpose/FormQuestionType enums) + `Conference.form(purpose)`; active-only ordered; null when unconfigured. 7 tests. -- [x] **▣ CHECKPOINT 2** — full suite 1197 green; additive-only schema change; PR #4707 open (stacked on #4705). - -## PR3 — grants backend (`generic-forms/03-grants-backend`) -- [ ] **T3.1** `Grant.form_answer` OneToOne (SET_NULL) + `blank=True` on the 4 required soft columns (`why`, `python_usage`, `been_to_other_events`, `occupation`); one migration. Columns stay NOT NULL — mutations must never pass `None`. -- [ ] **T3.2** Mutations + tests (TDD): 8 soft fields optional + optional `answers: JSON`; legacy soft-field checks run only when `answers` absent (structured-field validation unchanged on both paths); errors via `answers_errors: JSON` **direct assignment** (dotted paths impossible — verified); FormAnswer `update_or_create` in existing transaction; `None`→`""` coalescing on create; setattr skip-list on update. Named tests: **answers-only payload (exact PR5 shape)**, legacy-shape regression, invalid→atomic rollback, no-form-configured rejected, deadline-closed unchanged, update-no-duplicate. -- [ ] **T3.3** `Grant.formAnswers: JSON|null` on GraphQL type + query test (`me.grant.formAnswers`). -- [ ] **▣ CHECKPOINT 3** — suite green; back-compat verified both directions; PR3 opened; **after merge: deploy to staging (workflow_dispatch) for PR5 codegen**. - -## PR4 — admin display + export (`generic-forms/04-admin`) -- [ ] **T4.1** GrantAdmin readonly Q/A display (`format_html`), empty state, no N+1. Verify: `pytest grants/tests/test_admin.py` + manual. -- [ ] **T4.2** `GrantResource` dynamic columns via `get_export_resource_kwargs` → `__init__` fields append (3.3.9 verified path, incl. export-form preview); historical export unchanged. Verify: resource tests. -- [ ] **▣ CHECKPOINT 4** — suite green; manual CSV export; PR4 opened. - -## PR5 — frontend (`generic-forms/05-frontend`) — start only after PR3 on staging -- [ ] **T5.1** `dynamic-form/` component + fragment + codegen; 6 types via styleguide + InputWrapper (mirror invitation-letter-form); errors from `answersErrors` map; component test. Verify: `pnpm codegen && pnpm test && pnpm build`. -- [ ] **T5.2** New-submission integration: fetch `form(GRANT)`, swap 8 hardcoded inputs for DynamicForm, `answers` in payload (drop legacy 8), **null-form guard blocks submission**, strip legacy validation selections from submit-grant.graphql, prune dead options.ts constants. Verify: pnpm test/build + manual submit. -- [ ] **T5.3** Edit flow: `formAnswers` into my-grant.graphql, prefill DynamicForm, strip legacy selections from edit documents. Verify: pnpm build + manual edit. -- [ ] **▣ CHECKPOINT 5 — FINAL** — spec §11 walked one-by-one; manual E2E (author → submit → edit → admin → export); **ops: GRANT form created in staging + production admin BEFORE merge; cutover before grants open**; follow-up ticketed as TWO PRs (frontend strip → soak → backend input removal).