Post on 29-Jul-2020
Pushing the ORMto its limits
👋 Hello, I’m SigurdDeveloper at Kolonial.no • github.com/ljodal/djangcon-eu-2019
Agenda
• Some tips and tricks
• Subqueries
• Constraints and indexes
• Window functions
• Customizing the ORM
Disclaimer:This talk is quite code heavy!
Useful tips and tricks
CustomQuerySet
and Manager
class OrderManager(models.Manager): def create_order( self, customer, products ): ...
class OrderQuerySet(QuerySet): def undelivered(self): return self.filter(is_delivered=False)
class Order(models.Model): ... objects = OrderManager.from_queryset( OrderQuerySet )()
Order.objects.create_order(customer=...) Order.objects.undelivered()
Inspecting queries
>>> orders = Order.objects.all() <OrderQuerySet ...>
>>> str(orders.query) SELECT ... FROM «orders_order"
>>> print(order.explain(verbose=True)) Seq Scan on public.orders_order (cost=0.00..28.10 rows=1810 width=17) Output: id, customer_id, created_at, is_shipped
Avoiding extra
queries
for order in Order.objects.all(): # This triggers one query per order print(order.customer.name) # This also triggers one query per order for line in order.lines.all(): print(line)
Order.objects.select_related('customer')
Order.objects.prefetch_related('lines')
Avoiding race conditions
with transaction.atomic(): product = ( Product.objects .select_for_update() .get(id=1) ) product.inventory -= 1 product.save()
Subqueries
Latest order
customers = Customer.objects.annotate( latest_order_time=Subquery( Order.objects.filter( customer=OuterRef('pk'), ).order_by( '-created_at' ).values( 'created_at' )[:1] ) )
>>> customers.first().latest_order_time 1
With aggregation
budgets = SalesTarget.objects.annotate( gross_total_sales=Subquery( Order.objects.filter( created_at__year=OuterRef('year'), created_at__month=OuterRef('month'), ).values_list( ExtractYear('created_at'), ExtractMonth('created_at') ).annotate( gross_total=Sum('lines__gross_amount'), ).value_lists( 'gross_total', ) ), )
>>> budgets.first().gross_total_sales 12.00
Aggregation without
grouping
id | week_day | lines__gross_amount 1 | 7 | 7.5 2 | 1 | 2.5 3 | 3 | 2.0
targets = SalesTarget.objects.annotate( weekend_revenue=Subquery( Order.objects.filter( created_at__week_day__in=[7, 1], ).values_list( Sum(‘lines__gross_amount'), ) ), )
>>> targest.first().weekend_revenue 7.50 # Oops, this is not what we wanted
Aggregation without
grouping
id | week_day | lines__gross_amount 1 | 7 | 7.5 2 | 1 | 2.5 3 | 3 | 2.0
targets = SalesTargets.objects.annotate( weekend_revenue=Subquery( Order.objects.filter( created_at__week_day__in=[7, 1], ).values_list( Func( 'lines__gross_amount', function='SUM', ), ) ), )
>>> targets.first().weekend_revenue 10.00
Custom constraints and indexes
Unique constraints
class SalesTarget(Model): year = models.IntegerField() month = models.IntegerField() target = models.DecimalField(...)
class Meta:
unique_together = [ ('year', 'month'), ]
Partial unique
constraint
class Order(Model): ...
class Meta:
constraints = [ UniqueConstraint( name='limit_pending_orders', fields=['customer', 'is_shipped'], condition=Q(is_shipped=False), ) ]
Check constraint
class MonthlyBudget(Model): ...
class Meta:
constraints = [ CheckConstraint( check=Q(month__in=range(1, 13)), name='check_valid_month', ) ]
Partial index
class Order(Model): ...
class Meta:
indexes = [ Index( name='unshipped_orders', fields=['pk', ], condition=Q(is_shipped=False), ) ]
Window functions
Previous order from
same customer
orders = Order.objects.annotate( prev_order_id=Window( expression=Lag('order_id', 1), partition_by=[F('customer_id')], order_by=F('created_at').asc(), ), )
>>> orders.first().prev_order_id 1
Extending with custom functionality
Custom functions
class Round(Func): function = 'ROUND'
A more complex custom function
class AsDateTime(Func):
arity = 2 output_field = DateTimeField()
def as_postgresql( self, compiler, connection, **extra_context ):
extra_context['tz'] = settings.TIME_ZONE template = ( "(%(expressions)s || '%(tz)s')::timestamptz" )
return self.as_sql( compiler, connection, arg_joiner='+', template=template, **extra_context )
qs.annotate( datetime=AsDateTime('date_field', 'time_field') )
Writing custom SQL
Order.objects.annotate( age=RawSQL('age(created_at)'), )
Order.objects.extra( select={ 'age': 'age(created_at)', }, )
Order.objects.raw( ''' SELECT *, age(created_at) as age FROM orders_order ''' )
with connection.cursor() as cursor: cursor.execute('SELECT 2') cursor.fetchone()
There’s so much moreTake a look at the Django documentation
Thanks!@sigurdlj • github.com/ljodal/djangocon-eu-2019
medium.com/kolonial-no-product-tech