-
Notifications
You must be signed in to change notification settings - Fork 0
/
user_guide.html
2878 lines (2784 loc) · 334 KB
/
user_guide.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Fritz User Guide</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/theme.css " type="text/css" />
<link rel="stylesheet" href="_static/custom.css" type="text/css" />
<!-- sphinx script_files -->
<script src="_static/documentation_options.js?v=5929fcd5"></script>
<script src="_static/doctools.js?v=9a2dae69"></script>
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
<!-- bundled in js (rollup iife) -->
<!-- <script src="_static/theme-vendors.js"></script> -->
<script src="_static/theme.js" defer></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="Run your own Fritz" href="run_your_own.html" />
<link rel="prev" title="Fritz marshal manual" href="index.html" />
</head>
<body>
<div id="app">
<div class="theme-container" :class="pageClasses"><navbar @toggle-sidebar="toggleSidebar">
<router-link to="index.html" class="home-link">
<span class="site-name">Fritz Marshal</span>
</router-link>
<div class="links">
<navlinks class="can-hide">
</navlinks>
</div>
</navbar>
<div class="sidebar-mask" @click="toggleSidebar(false)">
</div>
<sidebar @toggle-sidebar="toggleSidebar">
<navlinks>
</navlinks><div id="searchbox" class="searchbox" role="search">
<div class="caption"><span class="caption-text">Quick search</span>
<div class="searchformwrapper">
<form class="search" action="search.html" method="get">
<input type="text" name="q" />
<input type="submit" value="Search" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
</div><div class="sidebar-links" role="navigation" aria-label="main navigation">
<div class="sidebar-group">
<p class="caption">
<span class="caption-text"><a href="index.html#fritz-marshal-manual">fritz marshal manual</a></span>
</p>
<ul class="current">
<li class="toctree-l1 current">
<a href="#" class="reference internal current">Fritz User Guide</a>
<ul>
<li class="toctree-l2"><a href="#quick-start" class="reference internal">Quick start</a></li>
<li class="toctree-l2"><a href="#alert-filters-in-fritz" class="reference internal">Alert filters in Fritz</a></li>
<li class="toctree-l2"><a href="#feedback" class="reference internal">Feedback</a></li>
</ul>
</li>
<li class="toctree-l1 ">
<a href="run_your_own.html" class="reference internal ">Run your own Fritz</a>
</li>
<li class="toctree-l1 ">
<a href="api.html" class="reference internal ">API</a>
</li>
<li class="toctree-l1 ">
<a href="catalogs.html" class="reference internal ">Kowalski collection schemas</a>
</li>
<li class="toctree-l1 ">
<a href="technical_implementation.html" class="reference internal ">Technical implementation</a>
</li>
<li class="toctree-l1 ">
<a href="developer.html" class="reference internal ">Developer Guidelines</a>
</li>
<li class="toctree-l1 ">
<a href="license.html" class="reference internal ">License</a>
</li>
</ul>
</div>
</div>
</sidebar>
<page>
<div class="body-header" role="navigation" aria-label="navigation">
<ul class="breadcrumbs">
<li><a href="index.html">Docs</a> »</li>
<li>Fritz User Guide</li>
</ul>
<ul class="page-nav">
<li class="prev">
<a href="index.html"
title="previous chapter">← Fritz marshal manual</a>
</li>
<li class="next">
<a href="run_your_own.html"
title="next chapter">Run your own Fritz →</a>
</li>
</ul>
</div>
<hr>
<div class="content" role="main" v-pre>
<section id="fritz-user-guide">
<h1>Fritz User Guide<a class="headerlink" href="#fritz-user-guide" title="Link to this heading">¶</a></h1>
<p><a class="reference external" href="https://github.com/fritz-marshal/fritz">Fritz</a> is the science data platform for the
<a class="reference external" href="https://ztf.caltech.edu">Zwicky Transient Facility (ZTF)</a> Phase II.</p>
<p>It implements an end-to-end, scalable, API-first system for Time-domain Astronomy, featuring</p>
<ul class="simple">
<li><p>A multi-survey data archive and alert broker</p></li>
<li><p>An interactive collaborative marshal for the transient, variable, and Solar system science cases</p></li>
<li><p>A workhorse for machine learning applications and active learning</p></li>
<li><p>Follow-up observation management with robotic and classical facilities</p></li>
<li><p>Fine-grained access control</p></li>
</ul>
<p>The key characteristics of Fritz are efficiency, scalability, portability, and extensibility.
Fritz employs a modular architecture and
integrates and extends two major components: <a class="reference external" href="https://github.com/dmitryduev/kowalski">Kowalski</a>
acts as the alert processor and data archive, and <a class="reference external" href="https://github.com/skyportal/skyportal">SkyPortal</a>,
which handles the rest of the stack.
The schematic overview of our system is shown below:</p>
<p><img alt="img/fritz.png" src="_images/fritz.jpg" /></p>
<section id="quick-start">
<h2>Quick start<a class="headerlink" href="#quick-start" title="Link to this heading">¶</a></h2>
<section id="a-quick-tour-of-fritz-from-the-user-perspective">
<h3>A quick tour of Fritz from the user perspective<a class="headerlink" href="#a-quick-tour-of-fritz-from-the-user-perspective" title="Link to this heading">¶</a></h3>
<section id="user-profile">
<h4>User profile<a class="headerlink" href="#user-profile" title="Link to this heading">¶</a></h4>
<p>Newly invited <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> users are first taken to the Profile page.
Here, you can update your user info, preferences, and manage tokens to
<a class="reference external" href="https://docs.fritz.science/api.html">interact with <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> programmatically</a>.
If you have a <a class="reference external" href="https://en.gravatar.com/">gravatar</a> associated with your email address,
it will be used as your avatar throughout the portal.</p>
<p><img alt="lsst-ws-profile" src="https://user-images.githubusercontent.com/7557205/113780243-cb733600-96e3-11eb-8f9d-3ed5734da11b.gif" /></p>
<p>System administrators can invite new users to <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> and manage their roles/ACLs as well as group membership and
alert stream access.</p>
<p>We use OAuth for user authentication.</p>
</section>
<section id="dashboard">
<h4>Dashboard<a class="headerlink" href="#dashboard" title="Link to this heading">¶</a></h4>
<p>On the Dashboard, the landing page, the user can see a collection of configurable widgets displaying the
information on the most popular and latest saved sources, the newsfeed, conditions at telescopes and so on.</p>
<p><img alt="lsst-ws-dashboard" src="https://user-images.githubusercontent.com/7557205/113781577-c0b9a080-96e5-11eb-8f08-5dd59c00cf01.gif" /></p>
</section>
<section id="groups-and-filters">
<h4>Groups and filters<a class="headerlink" href="#groups-and-filters" title="Link to this heading">¶</a></h4>
<p>New <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> users are added to one or more groups.
Users can also create groups themselves and request admission to groups they are not a member of.
<img alt="lsst-ws-group-admission" src="https://user-images.githubusercontent.com/7557205/113792878-d20da780-96fb-11eb-8e5d-359bd0e34807.gif" /></p>
<p>You can find and manage the list of group members, the sources saved to
the group, and the group’s <a class="reference external" href="https://docs.fritz.science/user_guide.html#alert-filters-in-fritz">alert filters</a>
(defined on one of the alert streams that the group has access to)
on the Group page. Group admins can create new filters (and modify existing ones).
System administrators can grant alert stream access to the groups.</p>
<p><code class="docutils literal notranslate"><span class="pre">Fritz</span></code> provides rich alert stream filtering capabilities through its Kowalski backend,
which consumes the ZTF Kafka alert stream, persisting the alerts to a database,
and supplementing them with additional data such as the Galactic coordinates,
external catalog cross-matches, machine learning scores etc.</p>
<p>Alert filters are implemented as <a class="reference external" href="https://docs.mongodb.com/manual/core/aggregation-pipeline/">aggregation pipelines</a>
and are executed on the enhanced packets containing, in particular, the full photometry history.
For a detailed discussion of this, refer to
the <a class="reference external" href="https://docs.fritz.science/user_guide.html#alert-filters-in-fritz">alert filters</a> section of the docs.</p>
<p>Fritz performs automated checks of the filter definition such that no code audit is necessary. Valid changes are propagated
and applied almost immediately. Users can enable/disable filters, inspect filter version history, opt in for
automatically saving passing objects to the filter’s group and updating annotations each time an object passes the filter.</p>
<p><img alt="lsst-ws-group-filter" src="https://user-images.githubusercontent.com/7557205/113790766-fe72f500-96f6-11eb-9c9a-7a7fc1ce8278.gif" /></p>
<p>Filters can range from very simple that rely on, for example,
<a class="reference external" href="https://docs.fritz.science/user_guide.html#acai-hosted-filter">mostly ML scores</a>,
or implement <a class="reference external" href="https://docs.fritz.science/user_guide.html#bts-rcf-program-full-filter">very complicated logic/computations</a>.
<a class="reference external" href="https://docs.fritz.science/user_guide.html#watch-lists">Watch lists</a> can also be implemented as simple Fritz filters.</p>
<p>Finally, we provide public alert databases for filter design and debugging.</p>
</section>
<section id="candidates-and-sources">
<h4>Candidates and Sources<a class="headerlink" href="#candidates-and-sources" title="Link to this heading">¶</a></h4>
<p>Alerts passing a filter are posted to Fritz’s SkyPortal backend as Candidates
and appear on the Scanning page (for the corresponding filter groups).</p>
<p>Candidates do not have to originate from <code class="docutils literal notranslate"><span class="pre">Kowalski</span></code> and could be posted (manually) via the API.</p>
<p>On the Candidates page, the users can filter, scan and inspect the objects that have passed filters of their groups
and save them to one or more groups. Candidates that are not saved to any group within 7 days are removed from <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>.
Saved Candidates become Sources that are persisted indefinitely.</p>
<p><img alt="lsst-ws-candidates2" src="https://user-images.githubusercontent.com/7557205/113795061-d8525280-9700-11eb-9ab0-69c599554ca5.gif" /></p>
<p>Candidates/Sources can be supplemented with (structured) annotations, which is particularly useful for scanning:
<img alt="lsst-ws-candidates3" src="https://user-images.githubusercontent.com/7557205/113927408-4ac73f00-97a2-11eb-9a30-e2fc3c3c5e66.gif" /></p>
<p>The Source page aggregates all kinds of information related to an object:
<img alt="lsst-ws-source-overview" src="https://user-images.githubusercontent.com/7557205/113828809-a7424400-9739-11eb-86b6-5eb525d32c82.gif" /></p>
<p>In particular, users can generate finder charts and star lists for the source, check its observability,
add and inspect redshift data:
<img alt="lsst-ws-source-1" src="https://user-images.githubusercontent.com/7557205/113828984-d658b580-9739-11eb-9272-d6e15928f894.gif" /></p>
<p>Annotations provide a way to store structured data related to the source,
while comments can be used to enter text/data in free-form. Sources can be classified according to a taxonomy
(which users can post themselves provided they have the necessary ACL).
Interactive offset plot displays positional information.</p>
<p>Users can send notifications about the source to the group members, provided they opted in for receiving such.</p>
<p><img alt="lsst-ws-source-2" src="https://user-images.githubusercontent.com/7557205/113928363-89a9c480-97a3-11eb-86d0-713b7c3ceee9.gif" /></p>
<p>For sources <a class="reference external" href="https://skyportal.io/docs/advanced_usage.html?highlight=gaia">annotated with Gaia color information</a>, an
HR diagram is rendered.
<img alt="lsst-ws-source-hr" src="https://user-images.githubusercontent.com/7557205/113984246-bab7e280-97ff-11eb-9c64-472af5864d64.gif" /></p>
<p>Users can interactively inspect (optionally binned) photometry data in both magnitude and flux spaces.
Tools for data import, export, and granular access management are available.
<img alt="lsst-ws-source-3" src="https://user-images.githubusercontent.com/7557205/113929129-882ccc00-97a4-11eb-86fb-e40fd7bb34ea.gif" /></p>
<p>The photometry plot for sources annotated with period data additionally displays phase-folded light curves:
<img alt="lsst-ws-source-phase-fold" src="https://user-images.githubusercontent.com/7557205/113984100-922fe880-97ff-11eb-92e3-47a78aab0595.gif" /></p>
<p>Tooling for extensive in-browser periodogram analysis for variable sources is also available:
<img alt="lsst-ws-source-periodogram-analysis" src="https://user-images.githubusercontent.com/7557205/113931157-00948c80-97a7-11eb-9125-1f7f9887718e.gif" /></p>
<p>Users can inspect, analyze, and manage (including export/import) spectroscopy data.</p>
<p>We provide interfaces to work with both robotic and classical follow-up facilities, including
a framework for dealing with allocations.
In particular, we feature automated integration with the <a class="reference external" href="https://sites.astro.caltech.edu/sedm/">SEDM</a>.</p>
<p><img alt="lsst-ws-source-spectroscopy-follow-up" src="https://user-images.githubusercontent.com/7557205/113935794-ced1f480-97ab-11eb-8c48-ea60fb3aee03.gif" /></p>
<p>Sources can be added to (“classical”) observing runs:
<img alt="lsst-ws-source-observing-run" src="https://user-images.githubusercontent.com/7557205/113935874-ea3cff80-97ab-11eb-8705-7858febbe6d1.gif" /></p>
<p>Sources can be added to favorites:
<img alt="lsst-ws-fav" src="https://user-images.githubusercontent.com/7557205/113828656-824dd100-9739-11eb-8291-2c7b779889a3.gif" /></p>
<p>Users can query the objects that exist in Fritz’s SkyPortal on the Sources page:
<img alt="lsst-ws-sources" src="https://user-images.githubusercontent.com/7557205/113937545-2bceaa00-97ae-11eb-9e66-1fe19aa79a62.gif" /></p>
<p>Only the objects that have been posted to <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>’s <code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code> backend are saved in its database.
However, <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>’s users can access the entire archive of ZTF alerts (~390M as of June 2021) via the Alerts page:</p>
<p><img alt="alerts-20210601" src="https://user-images.githubusercontent.com/7557205/120433501-287c1880-c330-11eb-907f-9c5327b9e0aa.gif" /></p>
<p><code class="docutils literal notranslate"><span class="pre">Fritz</span></code>’s users also have access to the entire archive of photometric light curves of ZTF sources
(~4B as of June 2021) via the Archive page:</p>
<p><img alt="archive-20210601" src="https://user-images.githubusercontent.com/7557205/120433545-36319e00-c330-11eb-9399-e68146a05d8b.gif" /></p>
</section>
<section id="mobile-experience">
<h4>Mobile experience<a class="headerlink" href="#mobile-experience" title="Link to this heading">¶</a></h4>
<p>Fritz’s interfaces are mobile-friendly, so the app will work as expected on your phone or tablet:</p>
<table>
<tr>
<td><img alt="phone" src="https://user-images.githubusercontent.com/7557205/114100592-5f2a3b00-9879-11eb-9dbc-9a4cdc46df77.gif" /></td>
<td><img alt="tablet" src="https://user-images.githubusercontent.com/7557205/114100803-b29c8900-9879-11eb-98c2-98ee91fd0e5e.gif" /></td>
</tr>
</table>
</section>
</section>
<section id="using-the-api">
<h3>Using the API<a class="headerlink" href="#using-the-api" title="Link to this heading">¶</a></h3>
<p>An API enables access to most of the underlying functionality of Fritz/SkyPortal/Kowalski.
The workflows described above are all enabled by specific API calls.
The complete OpenAPI specification is available at <a class="reference external" href="https://docs.fritz.science/api.html">https://docs.fritz.science/api.html</a>.</p>
<p>To use the API, you will need a token that can be generated on the Profile page. Once you have that, you can
access Fritz programmatically as follows.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Mapping</span><span class="p">,</span> <span class="n">Optional</span>
<span class="kn">import</span> <span class="nn">urllib.parse</span>
<span class="n">token</span> <span class="o">=</span> <span class="s1">'ea70a5f0-b321-43c6-96a1-b2de225e0339'</span>
<span class="k">def</span> <span class="nf">api</span><span class="p">(</span>
<span class="n">method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">endpoint</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<span class="n">data</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Mapping</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://fritz.science"</span><span class="p">,</span>
<span class="p">):</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"Authorization"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"token </span><span class="si">{</span><span class="n">token</span><span class="si">}</span><span class="s2">"</span><span class="p">}</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">request</span><span class="p">(</span>
<span class="n">method</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span>
<span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urljoin</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">endpoint</span><span class="p">),</span>
<span class="n">json</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">response</span>
</pre></div>
</div>
<p>For example, here is how to
<a class="reference external" href="https://docs.fritz.science/api.html#tag/sources/paths/~1api~1sources~1obj_id~1annotations/get">retrieve object’s annotations</a>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">object_id</span> <span class="o">=</span> <span class="s2">"ZTF21aatjavc"</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">api</span><span class="p">(</span><span class="s2">"get"</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"api/sources/</span><span class="si">{</span><span class="n">object_id</span><span class="si">}</span><span class="s2">/annotations"</span><span class="p">)</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"data"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
</pre></div>
</div>
<p>Which would yield something similar to:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span>
<span class="s2">"author_id"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
<span class="s2">"created_at"</span><span class="p">:</span> <span class="s2">"2021-04-08T06:17:56.980090"</span><span class="p">,</span>
<span class="s2">"data"</span><span class="p">:</span> <span class="p">{</span>
<span class="s2">"acai_b"</span><span class="p">:</span> <span class="mf">0.00011</span><span class="p">,</span>
<span class="s2">"acai_h"</span><span class="p">:</span> <span class="mf">0.98887</span><span class="p">,</span>
<span class="s2">"acai_n"</span><span class="p">:</span> <span class="mf">0.00361</span><span class="p">,</span>
<span class="s2">"acai_o"</span><span class="p">:</span> <span class="mf">0.00925</span><span class="p">,</span>
<span class="s2">"acai_v"</span><span class="p">:</span> <span class="mf">2e-05</span><span class="p">,</span>
<span class="s2">"age"</span><span class="p">:</span> <span class="mf">0.0</span><span class="p">,</span>
<span class="s2">"candid"</span><span class="p">:</span> <span class="s2">"1558254455015015000"</span><span class="p">,</span>
<span class="s2">"n_det"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">},</span>
<span class="s2">"id"</span><span class="p">:</span> <span class="mi">916700</span><span class="p">,</span>
<span class="s2">"modified"</span><span class="p">:</span> <span class="s2">"2021-04-08T06:17:56.980090"</span><span class="p">,</span>
<span class="s2">"obj_id"</span><span class="p">:</span> <span class="s2">"ZTF21aatjavc"</span><span class="p">,</span>
<span class="s2">"origin"</span><span class="p">:</span> <span class="s2">"au-public:hosted"</span><span class="p">,</span>
<span class="p">}]</span>
</pre></div>
</div>
</section>
</section>
<section id="alert-filters-in-fritz">
<h2>Alert filters in <code class="docutils literal notranslate"><span class="pre">Fritz</span></code><a class="headerlink" href="#alert-filters-in-fritz" title="Link to this heading">¶</a></h2>
<p>This section describes how to define alert stream filters within <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> and provides some examples for reference.</p>
<section id="introduction">
<h3>Introduction<a class="headerlink" href="#introduction" title="Link to this heading">¶</a></h3>
<p><a class="reference external" href="https://github.com/ZwickyTransientFacility/ztf-avro-alert">ZTF alerts</a> are
<a class="reference external" href="https://iopscience.iop.org/article/10.1088/1538-3873/aae8ac/meta">generated at IPAC</a> based on difference
imaging analysis and are distributed to the world at low latency via a Kafka alert stream.
<code class="docutils literal notranslate"><span class="pre">Fritz</span></code>’s <code class="docutils literal notranslate"><span class="pre">Kowalski</span></code> backend consumes this stream, persisting the alerts to a <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> database,
and supplementing them with other useful quantities such as Galactic coordinates, external catalog cross-matches,
machine learning scores etc. Next, <code class="docutils literal notranslate"><span class="pre">Kowalski</span></code> executes a series of user-defined filters on each new (“enhanced”)
incoming alert accessible to the filter. Users create filters on the <code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code> frontend and they are executed
on the <code class="docutils literal notranslate"><span class="pre">Kowalski</span></code> backend. If an alert passes a filter, it is pushed up to <code class="docutils literal notranslate"><span class="pre">SkyPortal</span></code> and appears on a program’s
scanning page.</p>
<p>Note: for a detailed description of the ZTF alerts and their contents, please see
<a class="reference external" href="https://github.com/ZwickyTransientFacility/ztf-avro-alert">here</a>.</p>
<section id="implementation-of-filters-as-mongodb-aggregation-pipelines">
<h4>Implementation of Filters as MongoDB Aggregation Pipelines<a class="headerlink" href="#implementation-of-filters-as-mongodb-aggregation-pipelines" title="Link to this heading">¶</a></h4>
<p><code class="docutils literal notranslate"><span class="pre">Kowalski</span></code> uses <a class="reference external" href="https://mongodb.com"><code class="docutils literal notranslate"><span class="pre">MongoDB</span></code></a>, a document-based NoSQL database, on the backend.</p>
<ul class="simple">
<li><p>For a very brief introduction into <code class="docutils literal notranslate"><span class="pre">MongoDB</span></code>, we recommend watching
<a class="reference external" href="https://www.youtube.com/watch?v=EE8ZTQxa0AM">MongoDB in 5 Minutes with Eliot Horowitz</a>.</p></li>
<li><p>If you are familiar with relational databases, you may want to check out the
<a class="reference external" href="https://docs.mongodb.com/manual/reference/sql-comparison/">SQL to MongoDB Mapping Chart</a>.</p></li>
</ul>
<p><code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> database stores its data in “collections”. A collection holds one or more “documents”.
Documents are analogous to records or rows in a relational database table.
Each document has one or more “fields”; fields are similar to the columns in a relational database table.
<code class="docutils literal notranslate"><span class="pre">MongoDB</span></code> supports a rich query language, which we will make use of when working with the alert filters.</p>
<p>Alert filtering is implemented as a
<a class="reference external" href="https://docs.mongodb.com/manual/core/aggregation-pipeline/">MongoDB aggregation pipeline</a> that first “massages”
the newly ingested alert data such that the user’s filter deals with enhanced “packets” containing, for example,
longer photometry history (and not just the rolling 30-day window), cross-match data, and custom ML scores.</p>
<p>The <a class="reference external" href="https://docs.mongodb.com/manual/core/aggregation-pipeline/">MongoDB aggregation pipeline</a> is a framework
for data aggregation modeled on the concept of data processing pipelines. Documents enter a multi-stage pipeline
that transforms the documents into aggregated results.</p>
<p>An aggregation pipeline is represented as a list of dictionaries, each corresponding to a processing stage/step:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
<span class="p">{</span><span class="o"><</span><span class="n">stage_1</span><span class="o">></span><span class="p">},</span>
<span class="o">...</span>
<span class="p">{</span><span class="o"><</span><span class="n">stage_N</span><span class="o">></span><span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
<p>Documents pass through the stages in sequence. Different stages can appear multiple times in a pipeline.</p>
<p>As of MongoDB version 4.2, there are 30 different types of aggregation pipeline stages, please see
<a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/">the official documentation</a> for a detailed
description.</p>
<p>To manipulate documents, each stage uses
<a class="reference external" href="https://docs.mongodb.com/manual/meta/aggregation-quick-reference/#expressions">expressions</a>,
which can include field paths, literals, system variables, expression objects, and expression operators.
Expressions can be nested.</p>
</section>
<section id="testing-your-filter">
<h4>Testing your filter<a class="headerlink" href="#testing-your-filter" title="Link to this heading">¶</a></h4>
<p>To ease the process of writing and debugging the filters, we have set up two live <em>public</em> <code class="docutils literal notranslate"><span class="pre">MongoDB</span> <span class="pre">Atlas</span></code> databases
in the cloud:</p>
<ul class="simple">
<li><p>The first one contains a small curated set of ~300 sample public ZTF alerts originating from SNe, variable stars,
AGN, and bogus detections. The auxiliary information is limited to the detection history present in the alert packets.</p></li>
<li><p>The second one contains ~120,000 public ZTF alerts from July 6, 2020. The auxiliary information contains a ~100-day
history of detections (limited to reduce the test database size), cross-matches with external catalogs, and a few
additional computed quantities.</p></li>
</ul>
<p>We recommend to begin exploring filtering on the first database and then move onto the second one.</p>
<p>We will show how to use a tool called <a class="reference external" href="https://www.mongodb.com/try/download/compass">MongoDB Compass</a>
(the full version is now free) to construct and debug aggregation pipelines (aka alert filters), using the two cloud
databases, that can be then plugged into <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>.</p>
<p>Filters will be managed on a dedicated page on <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>. A detailed description of the interface and its capabilities
(including, for example, filter versioning and diff’ing) will be covered elsewhere – please stay tuned.</p>
<section id="mongodb-compass">
<h5>MongoDB Compass<a class="headerlink" href="#mongodb-compass" title="Link to this heading">¶</a></h5>
<p>Download and install MongoDB Compass for your system from <a class="reference external" href="https://www.mongodb.com/try/download/compass">here</a>.</p>
<p>The connection string to access the first sample public alert database:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>mongodb://ztf:[email protected]:27017,fritz-test-shard-00-01-uas9d.gcp.mongodb.net:27017,fritz-test-shard-00-02-uas9d.gcp.mongodb.net:27017/test?authSource=admin&replicaSet=fritz-test-shard-0&readPreference=primary&appname=MongoDB%20Compass&ssl=true
</pre></div>
</div>
<p>The connection string to access the second sample public alert database:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mongodb</span><span class="o">+</span><span class="n">srv</span><span class="p">:</span><span class="o">//</span><span class="n">ztf</span><span class="p">:</span><span class="n">FritzZwicky</span><span class="nd">@fritz</span><span class="o">-</span><span class="n">public</span><span class="o">-</span><span class="mf">20200706.</span><span class="n">uas9d</span><span class="o">.</span><span class="n">gcp</span><span class="o">.</span><span class="n">mongodb</span><span class="o">.</span><span class="n">net</span><span class="o">/</span><span class="n">kowalski</span>
</pre></div>
</div>
<p>Upon connection:</p>
<ul class="simple">
<li><p>Select the <code class="docutils literal notranslate"><span class="pre">kowalski</span></code> database</p></li>
<li><p>Select the <code class="docutils literal notranslate"><span class="pre">ZTF_alerts</span></code> collection</p></li>
<li><p>Go to the Aggregations tab</p></li>
<li><p>Click the down arrow located to the left of the “Collation” button and select “New Pipeline From Text”</p></li>
<li><p>Copy paste the contents of the provided JSON files with the sample filters (see below)</p></li>
</ul>
<p>Note that Compass shows the output of all stages interactively and also displays/explains any errors in the code,
which is extremely helpful for debugging and experimentation.</p>
<p>By default, Compass’ aggregation pipeline builder works in Sample Mode, showing up to 20 documents that pass any
given stage. You can change the default settings, including the default timeout by clicking the button
with a little gear next to the “Auto Preview” toggle switch.</p>
<p><img alt="fritz-filters-01" src="https://user-images.githubusercontent.com/7557205/87487417-6f66ff80-c5f2-11ea-8d7b-49c51b6a502d.gif" /></p>
<p>Note: the filter code that you will save on Fritz must be valid JSON, so, once you’re done with debugging,
when exporting a filter in Compass, select “NODE” under “Export Pipeline To:”, copy-paste into a text editor,
and then manually replace single quotes with double quotes. We also recommend using a text editor that can validate
JSON.</p>
</section>
</section>
<section id="upstream-aggregation-pipeline-stages">
<h4>“Upstream” aggregation pipeline stages<a class="headerlink" href="#upstream-aggregation-pipeline-stages" title="Link to this heading">¶</a></h4>
<p>[OPTIONAL READ]</p>
<p>The upstream “massaging” mentioned above is performed by <code class="docutils literal notranslate"><span class="pre">Fritz</span></code> for each alert and includes:</p>
<ul class="simple">
<li><p>Selecting the newly ingested alert from the <code class="docutils literal notranslate"><span class="pre">ZTF_alerts</span></code> collection by its <code class="docutils literal notranslate"><span class="pre">candid</span></code></p></li>
<li><p>Removing the image cutouts to reduce traffic</p></li>
<li><p>Joining the alert by its <code class="docutils literal notranslate"><span class="pre">objectId</span></code> with the corresponding entry in the <code class="docutils literal notranslate"><span class="pre">ZTF_alerts_aux</span></code> collection, which contains
the cross-matches, ML scores, computed quantities, and archival photometry / detection history</p></li>
</ul>
<p>The upstream stages also take care of the ACLs.</p>
<p><code class="docutils literal notranslate"><span class="pre">Fritz</span></code> uses the following four stages:</p>
<ul class="simple">
<li><p>The first <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/"><code class="docutils literal notranslate"><span class="pre">$match</span></code></a>
stage selects the alert by its candid and ensures the ACLs are respected.</p></li>
</ul>
<p>For example, for a program that has access to the partnership data:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$match"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"candid"</span><span class="p">:</span><span class="w"> </span><span class="mi">1282486310015015001</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"candidate.programid"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$in"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>TIP: When debugging your filter with Compass, you may want to play around with this stage: try selecting an object
that must have passed the filter (when using the second database, that must have happened on July 6, 2020) by
specifying <code class="docutils literal notranslate"><span class="pre">"objectId":</span> <span class="pre">"<ZTF</span> <span class="pre">object</span> <span class="pre">id>"</span></code>, or turning it off altogether to make Mongo look at all the alerts stored
in the sample database.</p>
<ul class="simple">
<li><p>The image cutouts are stored per alert, but generally not needed for the filtering purposes.
The <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/"><code class="docutils literal notranslate"><span class="pre">$project</span></code></a> stage
removes them:</p></li>
</ul>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$project"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"cutoutScience"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"cutoutTemplate"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"cutoutDifference"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<ul class="simple">
<li><p>Using the <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/"><code class="docutils literal notranslate"><span class="pre">$lookup</span></code></a> stage,
the alert data are joined with auxiliary data stored in the <code class="docutils literal notranslate"><span class="pre">ZTF_alerts_aux</span></code> collection, which uses the alert’s
<code class="docutils literal notranslate"><span class="pre">objectId</span></code>s as unique document identifiers (<code class="docutils literal notranslate"><span class="pre">_id</span></code> - a concept in MongoDB similar to primary keys in SQL):</p></li>
</ul>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$lookup"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"from"</span><span class="p">:</span><span class="w"> </span><span class="s2">"ZTF_alerts_aux"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"localField"</span><span class="p">:</span><span class="w"> </span><span class="s2">"objectId"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"foreignField"</span><span class="p">:</span><span class="w"> </span><span class="s2">"_id"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"as"</span><span class="p">:</span><span class="w"> </span><span class="s2">"aux"</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
</pre></div>
</div>
<ul class="simple">
<li><p>The final <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/"><code class="docutils literal notranslate"><span class="pre">$project</span></code></a> stage reshapes
the joined data for convenience, selects the last 365 days of photometry history (which is done having practical
considerations in mind and may be relaxed in the future),
and applies ACLs to the detection history stored in <code class="docutils literal notranslate"><span class="pre">prv_candidates</span></code>:</p></li>
</ul>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$project"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"cross_matches"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$arrayElemAt"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$aux.cross_matches"</span><span class="p">,</span>
<span class="w"> </span><span class="mi">0</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"prv_candidates"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$filter"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"input"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$arrayElemAt"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$aux.prv_candidates"</span><span class="p">,</span>
<span class="w"> </span><span class="mi">0</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"as"</span><span class="p">:</span><span class="w"> </span><span class="s2">"item"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"cond"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$and"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$in"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$$item.programid"</span><span class="p">,</span>
<span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="mi">2</span><span class="p">,</span>
<span class="w"> </span><span class="mi">3</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$subtract"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$candidate.jd"</span><span class="p">,</span>
<span class="w"> </span><span class="s2">"$$item.jd"</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="mi">365</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"schemavsn"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"publisher"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"objectId"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"candid"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"candidate"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"classifications"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"coordinates"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The user-defined filter stages then operate on the “enhanced” packets that look like this:</p>
<p><img alt="img/filter-04.png" src="_images/filter-04.png" /></p>
<p><code class="docutils literal notranslate"><span class="pre">Fritz</span></code> automatically prepends these stages to all user-defined filters. However, when constructing/debugging
filters in Compass, the users must take care of that – all the examples below come with the upstream stages prepended.</p>
</section>
<section id="limitations">
<h4>Limitations<a class="headerlink" href="#limitations" title="Link to this heading">¶</a></h4>
<p><code class="docutils literal notranslate"><span class="pre">$lookup</span></code>, <code class="docutils literal notranslate"><span class="pre">$unionWith</span></code>, <code class="docutils literal notranslate"><span class="pre">$out</span></code>, and <code class="docutils literal notranslate"><span class="pre">$merge</span></code> stages are not allowed in the user-defined part of the filters.</p>
</section>
<section id="alert-data-augmentation">
<h4>Alert data augmentation<a class="headerlink" href="#alert-data-augmentation" title="Link to this heading">¶</a></h4>
<p>Fritz’s Kowalski backend augments the alert data with the following: [as of January 2021]</p>
<ul class="simple">
<li><p>Galactic coordinates</p></li>
<li><p>Cross-matches with external catalogs:</p>
<ul>
<li><p>2MASS_PSC (all matches within 2”)</p></li>
<li><p>AllWISE (all matches within 2”)</p></li>
<li><p>GALEX (all matches within 2”)</p></li>
<li><p>Gaia_DR2 (all matches within 2”)</p></li>
<li><p>Gaia_EDR3 (all matches within 2”)</p></li>
<li><p>Gaia_DR2_WD (all matches within 2”)</p></li>
<li><p>IPHAS_DR2 (all matches within 2”)</p></li>
<li><p>LAMOST_DR5_v3 (all matches within 2”)</p></li>
<li><p>PS1_DR1 (all matches within 2”)</p></li>
<li><p>PS1_STRM (all matches within 2”)</p></li>
<li><p>galaxy_redshifts_20200522 (all matches within 2”)</p></li>
<li><p>CLU_20190625 (<a class="reference external" href="https://github.com/dmitryduev/kowalski/blob/master/kowalski/alert_broker_ztf.py#L351">“elliptical” matches with close galaxies using 3x their size</a>)</p></li>
</ul>
</li>
</ul>
<p>For the detailed description of the available catalogs, see <a class="reference internal" href="#catalogs.html"><span class="xref myst">here</span></a></p>
<ul class="simple">
<li><p>Machine learning scores:</p>
<ul>
<li><p><a class="reference external" href="https://academic.oup.com/mnras/article/489/3/3582/5554758"><code class="docutils literal notranslate"><span class="pre">braai</span></code></a> version <code class="docutils literal notranslate"><span class="pre">d6_m9</span></code> – real/bogus classifier</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_h</span></code> version <code class="docutils literal notranslate"><span class="pre">d1_dnn_20201130</span></code> – phenomenological classifier, “hosted”</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_o</span></code> version <code class="docutils literal notranslate"><span class="pre">d1_dnn_20201130</span></code> – phenomenological classifier, “orphan”</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_n</span></code> version <code class="docutils literal notranslate"><span class="pre">d1_dnn_20201130</span></code> – phenomenological classifier, “nuclear”</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_v</span></code> version <code class="docutils literal notranslate"><span class="pre">d1_dnn_20201130</span></code> – phenomenological classifier, “variable star”</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_b</span></code> version <code class="docutils literal notranslate"><span class="pre">d1_dnn_20201130</span></code> – phenomenological classifier, “bogus”</p></li>
</ul>
</li>
</ul>
<section id="acai">
<h5>ACAI<a class="headerlink" href="#acai" title="Link to this heading">¶</a></h5>
<p>In November 2020, we deployed a set of new phenomenological deep learning classifiers called
ACAI (Alert-Classifying AI; publication in prep.).
The system consists of 5 binary classifiers:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">acai_h</span></code> – “hosted” – genuine transient in the vicinity of a “host” with (some) morphology,
e.g. something one could call a galaxy; should catch SN, Novae etc</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_o</span></code> – “orphan” – a genuine orphan transient, i.e. there are no identifiable “hosts” in its vicinity;
catches asteroids and hostless (or with hosts that are too faint) transients</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_n</span></code> – “nuclear” – a genuine transient occurring in a galaxy/quasar nucleus; should catch AGN, TDEs, etc.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_v</span></code> – “variable star” – variable star</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">acai_b</span></code> – “bogus” – a new version of the real/bogus classifier; could be thought of as (1 - braai)</p></li>
</ul>
<p>Each classifier takes as input 25 features from the candidate section of a ZTF alert packet and a stack of
full-sized thumbnails (science/reference (template)/difference) and produces a score from 0 to 1:</p>
<p><img alt="img/acai_h.d1_dnn_20201130.png" src="_images/acai_h.d1_dnn_20201130.png" /></p>
<p>All classifiers were trained on more than 200,000 diverse alerts covering a large part of the phase space.
The classifiers, although potentially correlated in their output, act independently and so can be
mixed and matched and applied alongside other alert features.</p>
</section>
</section>
</section>
<section id="filter-examples">
<h3>Filter examples<a class="headerlink" href="#filter-examples" title="Link to this heading">¶</a></h3>
<p>Let us explore some examples! The provided JSON files contain prepended upstream stages, but they are not shown in the
text below for brevity.</p>
<section id="simple-filter">
<h4>Simple filter<a class="headerlink" href="#simple-filter" title="Link to this heading">¶</a></h4>
<p>Let us start with a very simplistic example:</p>
<p><a class="reference download internal" download="" href="_downloads/3a45ff49357d80c6aaf7e59d4f9a6ecc/fritz_filter_101.json"><code class="xref download docutils literal notranslate"><span class="pre">fritz_filter_101.json</span></code></a></p>
<p>The first user-defined <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/"><code class="docutils literal notranslate"><span class="pre">$match</span></code></a>
stage selects alerts with deep-learning-based real-bogus scores (<code class="docutils literal notranslate"><span class="pre">candidate.drb</span></code>) greater than 0.9999999
that don’t have any matches with the <code class="docutils literal notranslate"><span class="pre">Gaia_DR2</span></code> catalog
(the zeroth element of the corresponding array does not exist)</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$match"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"candidate.drb"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$gt"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.9999999</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"cross_matches.Gaia_DR2.0"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$exists"</span><span class="p">:</span><span class="w"> </span><span class="kc">false</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>In case an alert passes the first stage, the output stage adds the annotations dictionary:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$project"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"_id"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"candid"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"objectId"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"annotations.author"</span><span class="p">:</span><span class="w"> </span><span class="s2">"dd"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"annotations.mean_rb"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$avg"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$prv_candidates.rb"</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">candid</span></code> and <code class="docutils literal notranslate"><span class="pre">objectId</span></code> are kept here for the debugging purposes, to see which alerts pass your filter.</p>
<p>The output of the filter will look something like this:</p>
<p><img alt="img/filter-05.png" src="_images/filter-05.png" /></p>
<p>This alert will be posted to the candidates page with these annotations.</p>
</section>
<section id="acai-hosted-filter">
<h4>ACAI-hosted filter<a class="headerlink" href="#acai-hosted-filter" title="Link to this heading">¶</a></h4>
<p>Let us build a filter that primarily relies on the ACAI ML models to select transients that are confidently
classified as “hosted” (and nothing else), and are a positive subtraction (<code class="docutils literal notranslate"><span class="pre">candidate.isdiffpos</span></code>) and
not a known Solar system object (<code class="docutils literal notranslate"><span class="pre">candidate.ssdistnr</span></code>):</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$match"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"classifications.acai_h"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$gte"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.8</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"classifications.acai_b"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.1</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"classifications.acai_v"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.1</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"classifications.acai_o"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.4</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"classifications.acai_n"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.4</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"candidate.ssdistnr"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"candidate.isdiffpos"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$in"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w"> </span><span class="kc">true</span><span class="p">,</span><span class="w"> </span><span class="s2">"t"</span><span class="p">]}</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$project"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"annotations.age"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[{</span><span class="nt">"$subtract"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$candidate.jd"</span><span class="p">,</span><span class="w"> </span><span class="s2">"$candidate.jdstarthist"</span><span class="p">]},</span><span class="w"> </span><span class="mi">5</span><span class="p">]},</span>
<span class="w"> </span><span class="nt">"annotations.n_det"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.ndethist"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"annotations.candid"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$toString"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candid"</span><span class="p">},</span>
<span class="w"> </span><span class="nt">"annotations.acai_h"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$classifications.acai_h"</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">]},</span>
<span class="w"> </span><span class="nt">"annotations.acai_v"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$classifications.acai_v"</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">]},</span>
<span class="w"> </span><span class="nt">"annotations.acai_o"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$classifications.acai_o"</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">]},</span>
<span class="w"> </span><span class="nt">"annotations.acai_n"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$classifications.acai_n"</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">]},</span>
<span class="w"> </span><span class="nt">"annotations.acai_b"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">"$round"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="s2">"$classifications.acai_b"</span><span class="p">,</span><span class="w"> </span><span class="mi">5</span><span class="p">]}</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
</section>
<section id="clu-filter">
<h4>CLU filter<a class="headerlink" href="#clu-filter" title="Link to this heading">¶</a></h4>
<p>Now that we’ve looked at basic examples, let us explore a more complex case and build a filter for
the <a class="reference external" href="https://ui.adsabs.harvard.edu/abs/2020arXiv200409029D/abstract">Census of the Local Universe</a> program.</p>
<p>As a reference, we will use the filter definition (as of July 10, 2020)
from the GROWTH marshal translated into <code class="docutils literal notranslate"><span class="pre">python</span></code> code (copy-pasted here with no alterations):</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">clu_filter</span><span class="p">(</span><span class="n">current_observation</span><span class="p">):</span>
<span class="n">filteron</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">annotations</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">calccount</span> <span class="o">=</span> <span class="mi">10000</span>
<span class="n">bright</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">nopointunderneath</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">mover</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">real</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">tdiff</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mf">99.0</span><span class="p">)</span>
<span class="n">magdiff</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mi">99</span><span class="p">)</span>
<span class="n">riserate</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mi">99</span><span class="p">)</span>
<span class="n">decayrate</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mi">99</span><span class="p">)</span>
<span class="n">hostgr</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mi">99</span><span class="p">)</span>
<span class="n">hostri</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="mi">99</span><span class="p">)</span>
<span class="n">positivesubtraction</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">brightstar</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">highlum</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">deltajd</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">prevcandidates</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'prv_candidates'</span><span class="p">]</span>
<span class="n">m_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">m_app</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magap'</span><span class="p">]</span>
<span class="n">t_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">fid_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'fid'</span><span class="p">]</span>
<span class="n">sgscore</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'sgscore1'</span><span class="p">]</span>
<span class="n">sgscore2</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'sgscore2'</span><span class="p">]</span>
<span class="n">sgscore3</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'sgscore3'</span><span class="p">]</span>
<span class="n">srmag</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'srmag1'</span><span class="p">]</span>
<span class="n">srmag2</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'srmag2'</span><span class="p">]</span>
<span class="n">srmag3</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'srmag3'</span><span class="p">]</span>
<span class="n">sgmag</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'sgmag1'</span><span class="p">]</span>
<span class="n">simag</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'simag1'</span><span class="p">]</span>
<span class="n">rbscore</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'rb'</span><span class="p">]</span>
<span class="n">magnr</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magnr'</span><span class="p">]</span>
<span class="n">distnr</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'distnr'</span><span class="p">]</span>
<span class="n">distpsnr1</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'distpsnr1'</span><span class="p">]</span>
<span class="n">distpsnr2</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'distpsnr2'</span><span class="p">]</span>
<span class="n">distpsnr3</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'distpsnr3'</span><span class="p">]</span>
<span class="n">scorr</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'scorr'</span><span class="p">]</span>
<span class="n">fwhm</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'fwhm'</span><span class="p">]</span>
<span class="n">elong</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'elong'</span><span class="p">]</span>
<span class="n">nbad</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'nbad'</span><span class="p">]</span>
<span class="n">chipsf</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'chipsf'</span><span class="p">]</span>
<span class="n">gal_lat</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'gal_lat'</span><span class="p">]</span>
<span class="n">jdstarthist</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jdstarthist'</span><span class="p">]</span>
<span class="n">jdendhist</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jdendhist'</span><span class="p">]</span>
<span class="n">ssdistnr</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'ssdistnr'</span><span class="p">]</span>
<span class="n">ssmagnr</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'ssmagnr'</span><span class="p">]</span>
<span class="n">drb</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'drb'</span><span class="p">]</span>
<span class="k">if</span> <span class="p">(</span><span class="n">jdstarthist</span> <span class="ow">and</span> <span class="n">jdendhist</span><span class="p">):</span>
<span class="n">deltajd</span> <span class="o">=</span> <span class="n">jdendhist</span> <span class="o">-</span> <span class="n">jdstarthist</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">psfminap</span> <span class="o">=</span> <span class="n">m_now</span> <span class="o">-</span> <span class="n">m_app</span>
<span class="n">bright</span> <span class="o">=</span> <span class="n">m_now</span> <span class="o"><</span> <span class="mf">99.0</span>
<span class="k">if</span> <span class="p">(</span><span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="ow">and</span> <span class="p">(</span><span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'t'</span> <span class="ow">or</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'1'</span><span class="p">)):</span>
<span class="n">positivesubtraction</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="p">(</span><span class="n">rbscore</span> <span class="ow">and</span> <span class="n">rbscore</span> <span class="o">></span> <span class="mf">0.3</span> <span class="ow">and</span> <span class="n">drb</span> <span class="o">></span> <span class="mf">0.5</span> <span class="ow">and</span> <span class="n">fwhm</span> <span class="o">></span> <span class="mf">0.5</span> <span class="ow">and</span> <span class="n">fwhm</span> <span class="o"><</span> <span class="mi">8</span> <span class="ow">and</span> <span class="n">nbad</span> <span class="o"><</span> <span class="mi">5</span> <span class="ow">and</span> <span class="p">(</span><span class="n">psfminap</span> <span class="o"><</span> <span class="mf">0.75</span> <span class="ow">or</span> <span class="n">psfminap</span> <span class="o">></span> <span class="p">(</span><span class="o">-</span><span class="mf">0.75</span><span class="p">))):</span>
<span class="n">real</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="p">(</span><span class="n">sgscore</span> <span class="ow">and</span> <span class="n">distpsnr1</span> <span class="ow">and</span> <span class="n">sgscore</span> <span class="o">></span> <span class="mf">0.76</span> <span class="ow">and</span> <span class="n">distpsnr1</span> <span class="o"><</span> <span class="mi">2</span><span class="p">):</span>
<span class="n">nopointunderneath</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="p">((</span><span class="n">distpsnr1</span> <span class="ow">and</span> <span class="n">srmag</span> <span class="ow">and</span> <span class="n">distpsnr1</span> <span class="o"><</span> <span class="mi">20</span> <span class="ow">and</span> <span class="n">srmag</span> <span class="o"><</span> <span class="mf">15.0</span> <span class="ow">and</span> <span class="n">srmag</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">sgscore</span> <span class="o">></span> <span class="mf">0.49</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="n">distpsnr2</span> <span class="ow">and</span> <span class="n">srmag2</span> <span class="ow">and</span> <span class="n">distpsnr2</span> <span class="o"><</span> <span class="mi">20</span> <span class="ow">and</span> <span class="n">srmag2</span> <span class="o"><</span> <span class="mf">15.0</span> <span class="ow">and</span> <span class="n">srmag2</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">sgscore2</span> <span class="o">></span> <span class="mf">0.49</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="n">distpsnr3</span> <span class="ow">and</span> <span class="n">srmag3</span> <span class="ow">and</span> <span class="n">distpsnr3</span> <span class="o"><</span> <span class="mi">20</span> <span class="ow">and</span> <span class="n">srmag3</span> <span class="o"><</span> <span class="mf">15.0</span> <span class="ow">and</span> <span class="n">srmag3</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">sgscore3</span> <span class="o">></span> <span class="mf">0.49</span><span class="p">)):</span>
<span class="n">brightstar</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">for</span> <span class="n">candidate</span> <span class="ow">in</span> <span class="n">prevcandidates</span><span class="p">:</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'jd'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'fid'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="ow">and</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'t'</span> <span class="ow">or</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'1'</span><span class="p">)):</span>
<span class="n">dt</span> <span class="o">=</span> <span class="n">t_now</span> <span class="o">-</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="k">if</span> <span class="p">(</span><span class="n">dt</span> <span class="o">></span> <span class="mf">0.02</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="o"><</span> <span class="mi">99</span> <span class="ow">and</span> <span class="p">(</span><span class="n">ssdistnr</span> <span class="o">></span> <span class="mi">2</span> <span class="ow">or</span> <span class="n">ssdistnr</span> <span class="o"><</span> <span class="p">(</span><span class="o">-</span><span class="mf">0.5</span><span class="p">))):</span>
<span class="n">mover</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">3</span>
<span class="k">if</span> <span class="n">calccount</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">break</span>
<span class="n">prevcandidates</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'prv_candidates'</span><span class="p">]</span>
<span class="n">m_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">m_max</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">m_min</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">t_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">t_max</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">t_min</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">fid_now</span> <span class="o">=</span> <span class="n">current_observation</span><span class="p">[</span><span class="s1">'candidate'</span><span class="p">][</span><span class="s1">'fid'</span><span class="p">]</span>
<span class="k">for</span> <span class="n">candidate</span> <span class="ow">in</span> <span class="n">prevcandidates</span><span class="p">:</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'jd'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'fid'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="ow">and</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'t'</span> <span class="ow">or</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'isdiffpos'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'1'</span><span class="p">)):</span>
<span class="k">if</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'fid'</span><span class="p">]</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'fid'</span><span class="p">]</span> <span class="o">==</span> <span class="n">fid_now</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="n">m_now</span> <span class="o"><</span> <span class="mi">99</span> <span class="ow">and</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="o"><</span> <span class="mi">99</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="o">></span> <span class="n">m_max</span><span class="p">):</span>
<span class="n">m_max</span> <span class="o">=</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">t_max</span> <span class="o">=</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">3</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="p">(</span><span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="o"><</span> <span class="n">m_min</span><span class="p">):</span>
<span class="n">m_min</span> <span class="o">=</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span>
<span class="n">t_min</span> <span class="o">=</span> <span class="n">candidate</span><span class="p">[</span><span class="s1">'jd'</span><span class="p">]</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">3</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">3</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">if</span> <span class="n">calccount</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">break</span>
<span class="n">tdiff</span> <span class="o">=</span> <span class="n">t_max</span> <span class="o">-</span> <span class="n">t_min</span>
<span class="n">magdiff</span> <span class="o">=</span> <span class="n">m_min</span> <span class="o">-</span> <span class="n">m_max</span>
<span class="k">if</span> <span class="p">(</span><span class="n">tdiff</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="n">tdiff</span> <span class="o">></span> <span class="mi">0</span><span class="p">):</span>
<span class="n">riserate</span> <span class="o">=</span> <span class="n">magdiff</span> <span class="o">/</span> <span class="n">tdiff</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">decayrate</span> <span class="o">=</span> <span class="p">(</span><span class="o">-</span><span class="n">magdiff</span><span class="p">)</span> <span class="o">/</span> <span class="n">tdiff</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">3</span>
<span class="k">if</span> <span class="p">(</span><span class="n">ssdistnr</span> <span class="o"><</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">ssdistnr</span> <span class="o">></span> <span class="p">(</span><span class="o">-</span><span class="mf">0.5</span><span class="p">)):</span>
<span class="n">mover</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">calccount</span> <span class="o">-=</span> <span class="mi">2</span>
<span class="n">hostgr</span> <span class="o">=</span> <span class="n">sgmag</span> <span class="o">-</span> <span class="n">srmag</span>
<span class="n">hostri</span> <span class="o">=</span> <span class="n">srmag</span> <span class="o">-</span> <span class="n">simag</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'FWHM'</span><span class="p">]</span> <span class="o">=</span> <span class="n">fwhm</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'host g-r'</span><span class="p">]</span> <span class="o">=</span> <span class="n">hostgr</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'host r-i'</span><span class="p">]</span> <span class="o">=</span> <span class="n">hostri</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'mag at max'</span><span class="p">]</span> <span class="o">=</span> <span class="n">m_max</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'time at max'</span><span class="p">]</span> <span class="o">=</span> <span class="n">t_max</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'min-mag'</span><span class="p">]</span> <span class="o">=</span> <span class="n">m_min</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'min-time'</span><span class="p">]</span> <span class="o">=</span> <span class="n">t_min</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'time difference'</span><span class="p">]</span> <span class="o">=</span> <span class="n">tdiff</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'mag diff'</span><span class="p">]</span> <span class="o">=</span> <span class="n">magdiff</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'rise rate'</span><span class="p">]</span> <span class="o">=</span> <span class="n">riserate</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'decay rate'</span><span class="p">]</span> <span class="o">=</span> <span class="n">decayrate</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'host ZTF ref PSF r-mag'</span><span class="p">]</span> <span class="o">=</span> <span class="n">magnr</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'PS1 psf r-mag'</span><span class="p">]</span> <span class="o">=</span> <span class="n">srmag</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'rb score'</span><span class="p">]</span> <span class="o">=</span> <span class="n">rbscore</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'sgscore1'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sgscore</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'ZOGI scorr'</span><span class="p">]</span> <span class="o">=</span> <span class="n">scorr</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'distpsnr1'</span><span class="p">]</span> <span class="o">=</span> <span class="n">distpsnr1</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'distpsnr2'</span><span class="p">]</span> <span class="o">=</span> <span class="n">distpsnr2</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'distpsnr3'</span><span class="p">]</span> <span class="o">=</span> <span class="n">distpsnr3</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'magpsf'</span><span class="p">]</span> <span class="o">=</span> <span class="n">m_now</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'elongation'</span><span class="p">]</span> <span class="o">=</span> <span class="n">elong</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'magap_min_magpsf'</span><span class="p">]</span> <span class="o">=</span> <span class="n">psfminap</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'gal_lat'</span><span class="p">]</span> <span class="o">=</span> <span class="n">gal_lat</span>
<span class="n">annotations</span><span class="p">[</span><span class="s1">'deltajd'</span><span class="p">]</span> <span class="o">=</span> <span class="n">deltajd</span>
<span class="n">filteron</span> <span class="o">=</span> <span class="n">bright</span> <span class="ow">and</span> <span class="n">nopointunderneath</span> <span class="ow">and</span> <span class="p">((</span><span class="ow">not</span> <span class="n">mover</span><span class="p">))</span> <span class="ow">and</span> <span class="n">real</span> <span class="ow">and</span> <span class="n">positivesubtraction</span> <span class="ow">and</span> <span class="p">((</span><span class="ow">not</span> <span class="n">brightstar</span><span class="p">))</span>
<span class="k">return</span> <span class="n">filteron</span><span class="p">,</span> <span class="n">annotations</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">Fritz</span></code>-implementation that can be loaded into Compass can be found here:</p>
<p><a class="reference download internal" download="" href="_downloads/4fd1b6e21733be8043f24c6d76fc0032/fritz_filter_clu.json"><code class="xref download docutils literal notranslate"><span class="pre">fritz_filter_clu.json</span></code></a></p>
<p>Let us explore it step-by-step and look at the individual user-defined stages, again omitting
the upstream part.</p>
<p>Note: try turning off the first stage of the pipeline that is pre-configured to select alerts by <code class="docutils literal notranslate"><span class="pre">objectId</span></code>
for demo purposes. When working with the 20200706 database, try a different <code class="docutils literal notranslate"><span class="pre">objectId</span></code>, for example <code class="docutils literal notranslate"><span class="pre">ZTF20abjoqjy</span></code>.</p>
<p>The first <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/match/"><code class="docutils literal notranslate"><span class="pre">$match</span></code></a> stage checks whether
the alert has at least one match with the CLU catalog:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$match"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"cross_matches.CLU_20190625.0"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$exists"</span><span class="p">:</span><span class="w"> </span><span class="kc">true</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>In the following <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/project/"><code class="docutils literal notranslate"><span class="pre">$project</span></code></a>
stage, we define the variables that will be used by the downstream stages.
Note how we are using the <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/subtract/"><code class="docutils literal notranslate"><span class="pre">$subtract</span></code></a>
operator for the <code class="docutils literal notranslate"><span class="pre">deltajd</span></code> and <code class="docutils literal notranslate"><span class="pre">psfminap</span></code> fields.
Also, the <code class="docutils literal notranslate"><span class="pre">candidates_fid</span></code> field concatenates the previous >3sigma detections in the same filter as <code class="docutils literal notranslate"><span class="pre">fid_now</span></code>
with the current photometric point, which we will use below.</p>
<p>For the full list of MongoDB’s aggregation pipeline operators,
see <a class="reference external" href="https://docs.mongodb.com/manual/reference/operator/aggregation/">here</a>.</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$project"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"_id"</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"candid"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"objectId"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"prv_candidates.jd"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"prv_candidates.magpsf"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"prv_candidates.fid"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"prv_candidates.isdiffpos"</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"isdiffpos"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.isdiffpos"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"m_now"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.magpsf"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"m_app"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.magap"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"t_now"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.jd"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"fid_now"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.fid"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"sgscore"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.sgscore1"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"sgscore2"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.sgscore2"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"sgscore3"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.sgscore3"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"srmag"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.srmag1"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"srmag2"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.srmag2"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"srmag3"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.srmag3"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"sgmag"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.sgmag1"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"simag"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.simag1"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"rbscore"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.rb"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"drb"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.drb"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"magnr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.magnr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"distnr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.distnr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"distpsnr1"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.distpsnr1"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"distpsnr2"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.distpsnr2"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"distpsnr3"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.distpsnr3"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"scorr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.scorr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"fwhm"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.fwhm"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"elong"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.elong"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"nbad"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.nbad"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"chipsf"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.chipsf"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"gal_lat"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$coordinates.b"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"ssdistnr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.ssdistnr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"ssmagnr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.ssmagnr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"ssnamenr"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.ssnamenr"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"jdstarthist"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.jdstarthist"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"jdendhist"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.jdendhist"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"deltajd"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$subtract"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$candidate.jdendhist"</span><span class="p">,</span><span class="w"> </span><span class="s2">"$candidate.jdstarthist"</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"psfminap"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$subtract"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$candidate.magpsf"</span><span class="p">,</span><span class="w"> </span><span class="s2">"$candidate.magap"</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="nt">"candidates_fid"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$concatArrays"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$filter"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"input"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$prv_candidates"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"as"</span><span class="p">:</span><span class="w"> </span><span class="s2">"cand"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"cond"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$and"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$eq"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$$cand.fid"</span><span class="p">,</span><span class="w"> </span><span class="s2">"$candidate.fid"</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$gt"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$$cand.magpsf"</span><span class="p">,</span><span class="w"> </span><span class="mi">0</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"$lt"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">"$$cand.magpsf"</span><span class="p">,</span><span class="w"> </span><span class="mi">99</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">"jd"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.jd"</span><span class="p">,</span>
<span class="w"> </span><span class="nt">"magpsf"</span><span class="p">:</span><span class="w"> </span><span class="s2">"$candidate.magpsf"</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>