\babel@toc {english}{}\relax 
\contentsline {chapter}{\numberline {1}Introduction}{1}{chapter.1}%
\contentsline {section}{\numberline {1.1}What exists in Parsl now?}{2}{section.1.1}%
\contentsline {section}{\numberline {1.2}Diagram}{3}{section.1.2}%
\contentsline {section}{\numberline {1.3}Concept: Universal personal logging}{4}{section.1.3}%
\contentsline {section}{\numberline {1.4}Target audience}{4}{section.1.4}%
\contentsline {section}{\numberline {1.5}Modularity}{5}{section.1.5}%
\contentsline {subsection}{\numberline {1.5.1}Modularity as a requirement for a rich research landscape}{5}{subsection.1.5.1}%
\contentsline {subsection}{\numberline {1.5.2}Hourglass model with several waists}{6}{subsection.1.5.2}%
\contentsline {section}{\numberline {1.6}High level structure of this project}{6}{section.1.6}%
\contentsline {chapter}{\numberline {2}The data model}{7}{chapter.2}%
\contentsline {section}{\numberline {2.1}Introduction to wide events}{7}{section.2.1}%
\contentsline {section}{\numberline {2.2}What exists now: Parsl python logs vs Parsl monitoring records}{7}{section.2.2}%
\contentsline {section}{\numberline {2.3}Optional and missing data in observability}{8}{section.2.3}%
\contentsline {section}{\numberline {2.4}Data types}{9}{section.2.4}%
\contentsline {section}{\numberline {2.5}Distributed state machines \sphinxhyphen {} parsl issue \#4021}{10}{section.2.5}%
\contentsline {section}{\numberline {2.6}commercial observability vendors}{10}{section.2.6}%
\contentsline {section}{\numberline {2.7}The argument for templating log messages}{11}{section.2.7}%
\contentsline {section}{\numberline {2.8}Objects and spans}{11}{section.2.8}%
\contentsline {section}{\numberline {2.9}Entity keys}{12}{section.2.9}%
\contentsline {section}{\numberline {2.10}Other components}{12}{section.2.10}%
\contentsline {chapter}{\numberline {3}Generating wide records}{15}{chapter.3}%
\contentsline {section}{\numberline {3.1}What exists now}{15}{section.3.1}%
\contentsline {subsection}{\numberline {3.1.1}Parsl}{15}{subsection.3.1.1}%
\contentsline {subsection}{\numberline {3.1.2}Academy}{16}{subsection.3.1.2}%
\contentsline {section}{\numberline {3.2}New Python Code for log generation}{16}{section.3.2}%
\contentsline {subsection}{\numberline {3.2.1}Python API on logging side}{17}{subsection.3.2.1}%
\contentsline {subsection}{\numberline {3.2.2}anonymous/temporary identified python objects}{17}{subsection.3.2.2}%
\contentsline {subsection}{\numberline {3.2.3}Contributed: Modifying academy to generate wide events}{18}{subsection.3.2.3}%
\contentsline {section}{\numberline {3.3}Translating non\sphinxhyphen {}wide\sphinxhyphen {}event sources}{18}{section.3.3}%
\contentsline {subsection}{\numberline {3.3.1}Using Parsl monitoring events as wide logs}{18}{subsection.3.3.1}%
\contentsline {subsection}{\numberline {3.3.2}Using Work Queue \sphinxstyleliteralintitle {\sphinxupquote {transaction\_log}} as a wide log source}{19}{subsection.3.3.2}%
\contentsline {section}{\numberline {3.4}Adventure: adding observability to a prototype: idris2interchange}{19}{section.3.4}%
\contentsline {section}{\numberline {3.5}Performance measurement of patch stack on 2025\sphinxhyphen {}10\sphinxhyphen {}27}{22}{section.3.5}%
\contentsline {section}{\numberline {3.6}Idea: Parsl resource monitoring on a host\sphinxhyphen {}wide basis}{23}{section.3.6}%
\contentsline {section}{\numberline {3.7}Idea: worker node dmesg}{23}{section.3.7}%
\contentsline {section}{\numberline {3.8}Idea: automatic instrumentation}{23}{section.3.8}%
\contentsline {chapter}{\numberline {4}Moving wide records around}{25}{chapter.4}%
\contentsline {section}{\numberline {4.1}Comparison to Parsl logging}{25}{section.4.1}%
\contentsline {section}{\numberline {4.2}Comparison to Parsl Monitoring}{26}{section.4.2}%
\contentsline {section}{\numberline {4.3}Python Configurability}{27}{section.4.3}%
\contentsline {section}{\numberline {4.4}Adventure: Wide records stored as JSON in files}{28}{section.4.4}%
\contentsline {section}{\numberline {4.5}Moving in realtime}{28}{section.4.5}%
\contentsline {section}{\numberline {4.6}Adventure: Diaspora Octopus}{29}{section.4.6}%
\contentsline {section}{\numberline {4.7}Idea: Chronolog}{29}{section.4.7}%
\contentsline {section}{\numberline {4.8}Adventure: pytest observing interchange variables}{30}{section.4.8}%
\contentsline {section}{\numberline {4.9}Adventure: Academy agents can report their own relevant logs via action}{30}{section.4.9}%
\contentsline {chapter}{\numberline {5}Analysing wide records}{31}{chapter.5}%
\contentsline {section}{\numberline {5.1}Adventure: All events for a task, in two aspects/presentations}{31}{section.5.1}%
\contentsline {subsection}{\numberline {5.1.1}Task events from the monitoring.db presentation}{32}{subsection.5.1.1}%
\contentsline {subsection}{\numberline {5.1.2}Task events from a JSON logging presentation}{35}{subsection.5.1.2}%
\contentsline {section}{\numberline {5.2}Adventure: The minimal change necessary to get htex task logs into the above task trace}{36}{section.5.2}%
\contentsline {section}{\numberline {5.3}Adventure: blog: Visualization for task prioritisation}{36}{section.5.3}%
\contentsline {section}{\numberline {5.4}Task flow logs through the whole system}{38}{section.5.4}%
\contentsline {section}{\numberline {5.5}Algebra of rearranging and querying wide events}{40}{section.5.5}%
\contentsline {subsection}{\numberline {5.5.1}functorial}{40}{subsection.5.5.1}%
\contentsline {subsection}{\numberline {5.5.2}non\sphinxhyphen {}functorial}{41}{subsection.5.5.2}%
\contentsline {subsubsection}{\numberline {5.5.2.1}keys imply key operator}{42}{subsubsection.5.5.2.1}%
\contentsline {section}{\numberline {5.6}Adventure: Browser UI}{42}{section.5.6}%
\contentsline {subsection}{\numberline {5.6.1}Idea: Streaming\sphinxhyphen {}fold web UI}{44}{subsection.5.6.1}%
\contentsline {section}{\numberline {5.7}python side query model}{44}{section.5.7}%
\contentsline {section}{\numberline {5.8}academy visualization}{44}{section.5.8}%
\contentsline {section}{\numberline {5.9}HTEX vs WQ questions}{45}{section.5.9}%
\contentsline {section}{\numberline {5.10}Other record storage systems}{45}{section.5.10}%
\contentsline {section}{\numberline {5.11}type checking event schemas}{45}{section.5.11}%
\contentsline {section}{\numberline {5.12}Writing out JSON events (or other formats) after performing query work}{46}{section.5.12}%
\contentsline {chapter}{\numberline {6}Adventure: Academy vs Globus Compute}{47}{chapter.6}%
\contentsline {section}{\numberline {6.1}getting started}{47}{section.6.1}%
\contentsline {section}{\numberline {6.2}Launching an academy agent}{48}{section.6.2}%
\contentsline {section}{\numberline {6.3}Looking at GC\sphinxhyphen {}endpoint\sphinxhyphen {}side academy logs}{50}{section.6.3}%
\contentsline {chapter}{\numberline {7}The rest}{53}{chapter.7}%
\contentsline {section}{\numberline {7.1}Debugging monitoring performance as part of developing this prototype}{53}{section.7.1}%
\contentsline {section}{\numberline {7.2}See also}{60}{section.7.2}%
\contentsline {section}{\numberline {7.3}wheres the bottleneck \sphinxhyphen {} visualization}{61}{section.7.3}%
\contentsline {section}{\numberline {7.4}Review of changes made so far to Parsl and Academy}{61}{section.7.4}%
\contentsline {section}{\numberline {7.5}Applying this approach for academy}{61}{section.7.5}%
\contentsline {chapter}{\numberline {8}Acknowledgements}{63}{chapter.8}%
\contentsline {chapter}{Index}{65}{section*.3}%