v3.4.1

CHANGELOG.md

@@ -5,7 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [UNRELEASED]
+## [3.4.1]
+
+### Modified
+- added actual severity column to SubjectCullAdmin
+- renamed 'Mice' to 'Subject' in views
+
+### Fixed
+- all alive filters depend on death date instead of cull
+- django 5.1 deprecation: CheckConstraint check -> condition
+- various filter typos, e.g. 'To be reduced' filter now works
+- improvements to notifications performance
+- fix log typo in delete_expired_notifications management command
+
+## [3.4.0]
+
+### Modified
+- moved prune_cortexlab.py to iblalyx repository
+
+### Fixed
+- removed test for removed subject death save logic
+- fixed command for dumping test database fixtures
 
 ### Added
 - in `alyx.misc` the one_cache command module contains utils to generate cache dataframes from sessions and datasets querysets

alyx/actions/admin.py

@@ -9,6 +9,7 @@
 from django.db.models import Case, When, Exists, OuterRef
 from django.urls import reverse
 from django.utils.html import format_html
+from django.utils import timezone
 from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter
 from django.contrib.admin import TabularInline
 from django.contrib.contenttypes.models import ContentType
@@ -63,9 +64,9 @@ def lookups(self, request, model_admin):
 
     def queryset(self, request, queryset):
         if self.value() is None:
-            return queryset.filter(subject__cull__isnull=True)
+            return queryset.exclude(subject__death_date__lte=timezone.now().date())
         if self.value() == 'n':
-            return queryset.exclude(subject__cull__isnull=True)
+            return queryset.filter(subject__death_date__lte=timezone.now().date())
         elif self.value == 'all':
             return queryset.all()
 
@@ -172,8 +173,13 @@ def __init__(self, *args, **kwargs):
         if 'subject' in self.fields and not (
                 self.current_user.is_stock_manager or self.current_user.is_superuser):
             inst = self.instance
-            ids = [s.id for s in Subject.objects.filter(responsible_user=self.current_user,
-                                                        cull__isnull=True).order_by('nickname')]
+            subjects = (Subject
+                        .objects
+                        .filter(responsible_user=self.current_user)
+                        .exclude(death_date__lte=timezone.now().date())
+                        .order_by('nickname')
+                        )
+            ids = list(subjects.values_list('pk', flat=True))
             if getattr(inst, 'subject', None):
                 ids = _bring_to_front(ids, inst.subject.pk)
             if getattr(self, 'last_subject_id', None):
@@ -184,8 +190,9 @@ def __init__(self, *args, **kwargs):
                 self.fields['subject'].queryset = Subject.objects.filter(
                     pk__in=ids).order_by(preserved, 'nickname')
             else:
-                self.fields['subject'].queryset = Subject.objects.filter(
-                    cull__isnull=True).order_by('nickname')
+                self.fields['subject'].queryset = (
+                    Subject.objects.exclude(death_date__lte=timezone.now().date()).order_by('nickname')
+                )
 
 
 class BaseActionAdmin(BaseAdmin):
@@ -349,7 +356,7 @@ class WaterRestrictionAdmin(BaseActionAdmin):
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == 'subject':
             obj = None
-            kwargs['queryset'] = Subject.objects.filter(cull__isnull=True).order_by('nickname')
+            kwargs['queryset'] = Subject.objects.exclude(death_date__lte=timezone.now().date()).order_by('nickname')
             # here if the form is of an existing water restriction, get the subject
             if request.resolver_match is not None:
                 object_id = request.resolver_match.kwargs.get('object_id')
@@ -493,7 +500,7 @@ def __init__(self, *args, **kwargs):
             # Order by alive subjects first
             ids = list(Subject
                        .objects
-                       .filter(death_date__isnull=True)
+                       .exclude(death_date__lte=timezone.now().date())
                        .order_by('nickname')
                        .only('pk')
                        .values_list('pk', flat=True))
@@ -762,6 +769,11 @@ class CullAdmin(BaseAdmin):
     fields = ('date', 'subject', 'user', 'cull_reason', 'cull_method', 'description')
     ordering = ('-date',)
 
+    def formfield_for_dbfield(self, db_field, request, **kwargs):
+        if db_field.name == 'date':
+            kwargs['initial'] = timezone.now().date()
+        return super(CullAdmin, self).formfield_for_dbfield(db_field, request, **kwargs)
+
     def subject_l(self, obj):
         url = get_admin_url(obj.subject)
         return format_html('<a href="{url}">{subject}</a>', subject=obj.subject or '-', url=url)

alyx/actions/management/commands/check_water_admin.py

@@ -16,6 +16,6 @@ def handle(self, *args, **options):
                 start_time__isnull=False,
                 end_time__isnull=True). \
             order_by('subject__responsible_user__username', 'subject__nickname')
-        for wr in wrs:
+        for wr in wrs.iterator():
             check_water_administration(wr.subject)
             check_weighed(wr.subject)

alyx/actions/management/commands/delete_expired_notifications.py

@@ -32,7 +32,7 @@ def handle(self, *args, **options):
             self.stdout.write(self.style.NOTICE(message))
         else:
             _, deleted_count = notifications.delete()
-            message = f'Deleted {deleted_count["actions.Notification"]} notifications'
+            message = f'Deleted {deleted_count.get("actions.Notification", 0)} notifications'
             if status:
                 message += f' with status "{status}"'
             self.stdout.write(self.style.SUCCESS(message))

alyx/actions/migrations/0024_surgery_implant_weight.py

@@ -19,6 +19,6 @@ class Migration(migrations.Migration):
         ),
         migrations.AddConstraint(
             model_name='surgery',
-            constraint=models.CheckConstraint(check=models.Q(('implant_weight__gte', 0)), name='implant_weight_gte_0'),
+            constraint=models.CheckConstraint(condition=models.Q(('implant_weight__gte', 0)), name='implant_weight_gte_0'),
         ),
     ]

alyx/actions/models.py

@@ -10,7 +10,7 @@
 from django.utils import timezone
 
 from alyx.base import BaseModel, modify_fields, alyx_mail, BaseManager
-from misc.models import Lab, LabLocation, LabMember, Note
+from misc.models import Lab, LabLocation, LabMember, LabMembership, Note
 
 
 logger = logging.getLogger(__name__)
@@ -62,6 +62,10 @@ def expected(self):
 
     def save(self, *args, **kwargs):
         super(Weighing, self).save(*args, **kwargs)
+        # Update subject water control
+        if self.subject._water_control:  # if already initialized
+            self.subject.water_control.add_weighing(self.date_time, self.weight)
+        # Check for underweight
         from actions.notifications import check_underweight
         check_underweight(self.subject)
 
@@ -212,7 +216,7 @@ class Meta:
         verbose_name_plural = "surgeries"
         constraints = [
             models.CheckConstraint(
-                check=models.Q(implant_weight__gte=0), name="implant_weight_gte_0"),
+                condition=models.Q(implant_weight__gte=0), name="implant_weight_gte_0"),
         ]
 
     def save(self, *args, **kwargs):
@@ -388,52 +392,64 @@ class OtherAction(BaseAction):
 def delay_since_last_notification(notification_type, title, subject):
     """Return the delay since the last notification corresponding to the given
     type, title, subject, in seconds, whether it was actually sent or not."""
-    last_notif = Notification.objects.filter(
-        notification_type=notification_type,
-        title=title,
-        subject=subject).exclude(status='no-send').order_by('send_at').last()
+    last_notif = (
+        Notification
+        .objects
+        .filter(
+            notification_type=notification_type,
+            title=title,
+            subject=subject)
+        .exclude(status='no-send')
+        .order_by('send_at')
+        .values_list('sent_at', 'send_at')
+        .last()
+    )
     if last_notif:
-        date = last_notif.sent_at or last_notif.send_at
+        date = last_notif[0] or last_notif[1]
         return (timezone.now() - date).total_seconds()
     return inf
 
 
 def check_scope(user, subject, scope):
-    if subject is None:
-        return True
     # Default scope: mine.
     scope = scope or 'mine'
     assert scope in ('none', 'mine', 'lab', 'all')
     if scope == 'mine':
-        return subject.responsible_user == user
+        return subject.responsible_user.pk == user
     elif scope == 'lab':
-        return subject.lab.name in (user.lab or ())
+        return (
+            LabMembership
+            .objects
+            .filter(lab=subject.lab, user=user, start_date__lt=timezone.now().date())
+            .exclude(end_date__lt=timezone.now().date())
+            .exists()
+        )
     elif scope == 'all':
         return True
-    elif scope == 'none':
-        return False
+    return False
 
 
 def get_recipients(notification_type, subject=None, users=None):
-    """Return the list of users that will receive a notification."""
+    """Return the set of users that will receive a notification."""
     # Default: initial list of recipients is the subject's responsible user.
     if users is None and subject and subject.responsible_user:
-        users = [subject.responsible_user]
-    if users is None:
-        users = []
+        users = {subject.responsible_user}
+    users = set() if users is None else set(users)
+    rules = dict(
+        NotificationRule
+        .objects
+        .select_related('user')
+        .filter(notification_type=notification_type)
+        .values_list('user', 'subjects_scope')
+    )
+    # Remove 'none' users from the specified users (those who explicitly unsubscribed).
+    users = users - {LabMember.objects.get(pk=user) for user, scope in rules.items() if scope == 'none'}
     if not subject:
         return users
-    members = LabMember.objects.all()
-    rules = NotificationRule.objects.filter(notification_type=notification_type)
-    # Dictionary giving the scope of every user in the database.
-    user_rules = {user: None for user in members}
-    user_rules.update({rule.user: rule.subjects_scope for rule in rules})
-    # Remove 'none' users from the specified users.
-    users = [user for user in users if user_rules.get(user, None) != 'none']
-    # Return the selected users, and those who opted in in the notification rules.
-    return users + [member for member in members
-                    if check_scope(member, subject, user_rules.get(member, None)) and
-                    member not in users]
+
+    # Include those who opted subscribed in the notification rules.
+    users |= {LabMember.objects.get(pk=user) for user, scope in rules.items() if check_scope(user, subject, scope)}
+    return users
 
 
 def create_notification(
@@ -463,7 +479,7 @@ def create_notification(
 def send_pending_emails(max_resend=3):
     """Send all pending notifications."""
     notifications = Notification.objects.filter(status='to-send', send_at__lte=timezone.now())
-    for notification in notifications:
+    for notification in notifications.iterator():
         success = notification.send_if_needed()
         # If not successful, and max_resend reached, set to no-send.
         if (

alyx/actions/notifications.py

@@ -20,7 +20,7 @@ def check_underweight(subject, date=None):
     """Called when a weighing is added."""
     # Reinit the water_control instance to make sure the just-added
     # weighing is taken into account
-    wc = subject.reinit_water_control()
+    wc = subject.water_control
     perc = wc.percentage_weight(date=date)
     min_perc = wc.min_percentage(date=date)
     lwb = wc.last_weighing_before(date=date)
@@ -36,7 +36,7 @@ def check_weighed(subject, date=None):
     date = date or timezone.now()
     # Reinit the water_control instance to make sure the just-added
     # weighing is taken into account
-    wc = subject.reinit_water_control()
+    wc = subject.water_control
     if not wc or not wc.is_water_restricted(date):
         return
 
@@ -73,7 +73,7 @@ def check_water_administration(subject, date=None):
         The datetime to check, deafults to now.
     """
     date = date or timezone.now()
-    wc = subject.reinit_water_control()
+    wc = subject.water_control
     if not wc or not wc.is_water_restricted(date):
         return
     remaining = wc.remaining_water(date=date)

alyx/actions/tests.py

@@ -175,6 +175,7 @@ def setUp(self):
             water_administered=10,
         )
         self.date = to_date('2018-06-10')
+        self.subject.reinit_water_control()
 
     def tearDown(self):
         base.DISABLE_MAIL = False

alyx/actions/water_control.py

@@ -5,7 +5,7 @@
 import functools
 import io
 import logging
-from operator import attrgetter, itemgetter
+from operator import itemgetter
 import os.path as op
 
 from django.conf import settings
@@ -615,30 +615,53 @@ def water_control(subject):
     if absolute_min := settings.WEIGHT_THRESHOLD:
         wc.add_threshold(
             percentage=absolute_min, bgcolor=PALETTE['red'], fgcolor='#F08699', line_style='--')
-    # Water restrictions.
-    wrs = sorted(list(subject.actions_waterrestrictions.all()), key=attrgetter('start_time'))
     # Surgeries.
-    srgs = sorted(list(subject.actions_surgerys.all()), key=attrgetter('start_time'))
-    for srg in srgs:
-        iw = srg.implant_weight
-        if iw:
-            wc.add_implant_weight(srg.start_time, iw)
+    srgs = (
+        subject
+        .actions_surgerys
+        .all()
+        .order_by('start_time')
+        .values_list('start_time', 'implant_weight')
+    )
+    for (start_time, implant_weight) in srgs.iterator():
+        if implant_weight:
+            wc.add_implant_weight(start_time, implant_weight)
+
+    # Water restrictions.
+    wrs = list(
+        subject
+        .actions_waterrestrictions
+        .all()
+        .order_by('start_time')
+        .values_list('start_time', 'reference_weight', 'end_time')
+    )
     # Reference weight.
     last_wr = wrs[-1] if wrs else None
-    if last_wr:
-        if last_wr.reference_weight:
-            wc.set_reference_weight(last_wr.start_time, last_wr.reference_weight)
-    for wr in wrs:
-        wc.add_water_restriction(wr.start_time, wr.end_time, wr.reference_weight)
+    if last_wr and last_wr[1]:  # reference weight
+        wc.set_reference_weight(last_wr[0], last_wr[1])
+    for (start_time, reference_weight, end_time) in wrs:
+        wc.add_water_restriction(start_time, end_time, reference_weight)
 
     # Water administrations.
-    was = sorted(list(subject.water_administrations.all()), key=attrgetter('date_time'))
-    for wa in was:
-        wc.add_water_administration(wa.date_time, wa.water_administered, session=wa.session_id)
+    was = (
+        subject
+        .water_administrations
+        .all()
+        .order_by('date_time')
+        .values_list('date_time', 'water_administered', 'session_id')
+    )
+    for (date_time, water_administered, session_id) in was.iterator():
+        wc.add_water_administration(date_time, water_administered, session=session_id)
 
     # Weighings
-    ws = sorted(list(subject.weighings.all()), key=attrgetter('date_time'))
-    for w in ws:
-        wc.add_weighing(w.date_time, w.weight)
+    ws = (
+        subject
+        .weighings
+        .all()
+        .order_by('date_time')
+        .values_list('date_time', 'weight')
+    )
+    for (date_time, weight) in ws.iterator():
+        wc.add_weighing(date_time, weight)
 
     return wc

alyx/alyx/__init__.py

@@ -1 +1 @@
-VERSION = __version__ = '3.4.0'
+VERSION = __version__ = '3.4.1'

alyx/misc/admin.py

@@ -9,6 +9,7 @@
 from django.contrib.postgres.fields import JSONField
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
+from django.utils import timezone
 from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter
 from rest_framework.authtoken.models import TokenProxy
 from rangefilter.filters import DateRangeFilter
@@ -202,7 +203,7 @@ class HousingSubjectAdminInline(admin.TabularInline):
 
     def get_queryset(self, request):
         qs = super(HousingSubjectAdminInline, self).get_queryset(request)
-        return qs.filter(subject__cull__isnull=True)
+        return qs.exclude(subject__death_date__lte=timezone.now().date())
 
 
 class HousingAdminForm(forms.ModelForm):