diff for duplicates of <ZdZOYnIkjqkyfo5P@x1> diff --git a/a/1.txt b/N1/1.txt index d12da54..2e23717 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -1,143 +1,127 @@ -In Thu, 4 Jan 2024 19:35:57 -0300, Arnaldo Carvalho de Melo wrote: -> Em Fri, Jul 28, 2023 at 05:07:18PM +0200, Sebastian Andrzej Siewior escreveu: -> > On 2023-07-26 08:10:45 [+0200], Mike Galbraith wrote: -> > > > [ 52.848925] BUG: scheduling while atomic: perf/6549/0x00000002 - -> > > Had bf9ad37dc8a not been reverted due to insufficient beauty, you could -> > > trivially make the sigtrap test a happy camper (wart tested in tip-rt). +> In Thu, 4 Jan 2024 19:35:57 -0300, Arnaldo Carvalho de Melo wrote: +> > +++ b/kernel/events/core.c +> > @@ -6801,7 +6801,7 @@ static void perf_pending_task(struct callback_head *head) +> > * If we 'fail' here, that's OK, it means recursion is already disabled +> > * and we won't recurse 'further'. +> > */ +> >- preempt_disable_notrace(); +> >+ migrate_disable(); +> > rctx = perf_swevent_get_recursion_context(); -> > Thank you for the pointer Mike. +> Pardon my ignorance, is it safe to call preempt_count() with preemption +> enabled on PREEMPT_RT, or at least in the context being discussed here? -> > I guess we need this preempt_disable_notrace() in perf_pending_task() -> > due to context accounting in get_recursion_context(). Would a -> > migrate_disable() be sufficient or could we send the signal outside of -> > the preempt-disabled block? - -> I got back to this, need to go again over all the callers of -> perf_swevent_get_recursion_context(), from the first quick glance there -> are other places with preempt_disable()/enable(), but doing just the -> switch to migrate disable/enable on perf_pending_task() makes this -> specific test to work: - -> [acme@nine linux]$ git log --oneline -5 -> 086dab66d504 (HEAD -> linux-rt-devel/linux-6.7.y-rt/send_sig_perf.fix, tag: v6.7-rc5-rt5, linux-rt-devel/linux-6.7.y-rt) v6.7-rc5-rt5 -> 29e0d951f39b printk: Update the printk series. -> 2308ecc8ce88 (tag: v6.7-rc5-rt4) v6.7-rc5-rt4 -> 10d5f3551216 Merge tag 'v6.7-rc5' into linux-6.7.y-rt -> a39b6ac3781d (tag: v6.7-rc5, linux-rt-devel/master, linux-rt-devel/linux-6.7.y) Linux 6.7-rc5 -> [acme@nine linux]$ git diff -> diff --git a/kernel/events/core.c b/kernel/events/core.c -> index c9d123e13b57..a9b9ef60f6b3 100644 -> --- a/kernel/events/core.c -> +++ b/kernel/events/core.c -> @@ -6801,7 +6801,7 @@ static void perf_pending_task(struct callback_head *head) -> * If we 'fail' here, that's OK, it means recursion is already disabled -> * and we won't recurse 'further'. -> */ ->- preempt_disable_notrace(); ->+ migrate_disable(); -> rctx = perf_swevent_get_recursion_context(); - -Pardon my ignorance, is it safe to call preempt_count() with preemption -enabled on PREEMPT_RT, or at least in the context being discussed here? - -Because: - - perf_swevent_get_recursion_context() - get_recursion_context() - interrupt_context_level() - preempt_count() - -And: - -int perf_swevent_get_recursion_context(void) -{ - struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); - - return get_recursion_context(swhash->recursion); -} - -> if (event->pending_work) { -> @@ -6812,7 +6812,7 @@ static void perf_pending_task(struct callback_head *head) +> Because: -> if (rctx >= 0) -> perf_swevent_put_recursion_context(rctx); -> - preempt_enable_notrace(); -> + migrate_enable(); +> perf_swevent_get_recursion_context() +> get_recursion_context() +> interrupt_context_level() +> preempt_count() -> put_event(event); -> } -> [acme@nine linux]$ uname -a -> Linux nine 6.7.0-rc5-rt5.sigtrap-fix-dirty #2 SMP PREEMPT_RT Thu Jan 4 18:11:44 -03 2024 x86_64 x86_64 x86_64 GNU/Linux -> [acme@nine linux]$ sudo su - -> [sudo] password for acme: -> [root@nine ~]# -> [root@nine ~]# perf test sigtrap -> 68: Sigtrap : Ok -> [root@nine ~]# -> [root@nine ~]# perf probe -L perf_pending_task -> <perf_pending_task@/home/acme/git/linux/kernel/events/core.c:0> -> 0 static void perf_pending_task(struct callback_head *head) -> { -> 2 struct perf_event *event = container_of(head, struct perf_event, pending_task); -> 3 int rctx; - -> /* -> * If we 'fail' here, that's OK, it means recursion is already disabled -> * and we won't recurse 'further'. -> */ -> migrate_disable(); -> 10 rctx = perf_swevent_get_recursion_context(); -> -> 12 if (event->pending_work) { -> 13 event->pending_work = 0; -> 14 perf_sigtrap(event); -> 15 local_dec(&event->ctx->nr_pending); -> } -> -> 18 if (rctx >= 0) -> 19 perf_swevent_put_recursion_context(rctx); -> 20 migrate_enable(); - -> 22 put_event(event); -> } - -> #ifdef CONFIG_GUEST_PERF_EVENTS - -> [root@nine ~]# perf probe perf_pending_task -> Added new event: -> probe:perf_pending_task (on perf_pending_task) - -> You can now use it in all perf tools, such as: - -> perf record -e probe:perf_pending_task -aR sleep 1 +> And: + +> int perf_swevent_get_recursion_context(void) +> { +> struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); +> +> return get_recursion_context(swhash->recursion); +> } -> [root@nine ~]# perf trace --max-events=1 -e probe:perf_pending_task/max-stack=6/ perf test sigtrap -> 68: Sigtrap : Ok -> 0.000 :9608/9608 probe:perf_pending_task(__probe_ip: -2064408784) -> perf_pending_task ([kernel.kallsyms]) -> task_work_run ([kernel.kallsyms]) -> exit_to_user_mode_loop ([kernel.kallsyms]) -> exit_to_user_mode_prepare ([kernel.kallsyms]) -> irqentry_exit_to_user_mode ([kernel.kallsyms]) -> asm_sysvec_irq_work ([kernel.kallsyms]) -> [root@nine ~]# +Seems to be enough because perf_pending_task is a irq_work callback and +that is guaranteed not to reentry? -> [root@nine ~]# head -5 /etc/os-release -> NAME="Red Hat Enterprise Linux" -> VERSION="9.2 (Plow)" -> ID="rhel" -> ID_LIKE="fedora" -> VERSION_ID="9.2" -> [root@nine ~]# +Artem's tests with a RHEL kernel seems to indicate that, ditto for my, +will test with upstream linux-6.8.y-rt. -> I did the test without the above patch and the original problem is -> reproduced. - -> > This is also used in perf_pending_irq() and on PREEMPT_RT this is -> > invoked from softirq context which is preemptible. +But there is a lot more happening in perf_sigtrap and I'm not sure if +the irq_work callback gets preempted we would not race with something +else. -Humm, and then when going thru perf_pending_irq() we don't hit that -scheduling on atomic. +Marco, Mike, ideas? - Arnaldo + +> > if (event->pending_work) { +> > @@ -6812,7 +6812,7 @@ static void perf_pending_task(struct callback_head *head) +> +> > if (rctx >= 0) +> > perf_swevent_put_recursion_context(rctx); +> > - preempt_enable_notrace(); +> > + migrate_enable(); +> +> > put_event(event); +> > } +> > [acme@nine linux]$ uname -a +> > Linux nine 6.7.0-rc5-rt5.sigtrap-fix-dirty #2 SMP PREEMPT_RT Thu Jan 4 18:11:44 -03 2024 x86_64 x86_64 x86_64 GNU/Linux +> > [acme@nine linux]$ sudo su - +> > [sudo] password for acme: +> > [root@nine ~]# +> > [root@nine ~]# perf test sigtrap +> > 68: Sigtrap : Ok +> > [root@nine ~]# +> > [root@nine ~]# perf probe -L perf_pending_task +> > <perf_pending_task@/home/acme/git/linux/kernel/events/core.c:0> +> > 0 static void perf_pending_task(struct callback_head *head) +> > { +> > 2 struct perf_event *event = container_of(head, struct perf_event, pending_task); +> > 3 int rctx; +> +> > /* +> > * If we 'fail' here, that's OK, it means recursion is already disabled +> > * and we won't recurse 'further'. +> > */ +> > migrate_disable(); +> > 10 rctx = perf_swevent_get_recursion_context(); +> > +> > 12 if (event->pending_work) { +> > 13 event->pending_work = 0; +> > 14 perf_sigtrap(event); +> > 15 local_dec(&event->ctx->nr_pending); +> > } +> > +> > 18 if (rctx >= 0) +> > 19 perf_swevent_put_recursion_context(rctx); +> > 20 migrate_enable(); +> +> > 22 put_event(event); +> > } +> +> > #ifdef CONFIG_GUEST_PERF_EVENTS +> +> > [root@nine ~]# perf probe perf_pending_task +> > Added new event: +> > probe:perf_pending_task (on perf_pending_task) +> +> > You can now use it in all perf tools, such as: +> +> > perf record -e probe:perf_pending_task -aR sleep 1 +> +> > [root@nine ~]# perf trace --max-events=1 -e probe:perf_pending_task/max-stack=6/ perf test sigtrap +> > 68: Sigtrap : Ok +> > 0.000 :9608/9608 probe:perf_pending_task(__probe_ip: -2064408784) +> > perf_pending_task ([kernel.kallsyms]) +> > task_work_run ([kernel.kallsyms]) +> > exit_to_user_mode_loop ([kernel.kallsyms]) +> > exit_to_user_mode_prepare ([kernel.kallsyms]) +> > irqentry_exit_to_user_mode ([kernel.kallsyms]) +> > asm_sysvec_irq_work ([kernel.kallsyms]) +> > [root@nine ~]# +> +> > [root@nine ~]# head -5 /etc/os-release +> > NAME="Red Hat Enterprise Linux" +> > VERSION="9.2 (Plow)" +> > ID="rhel" +> > ID_LIKE="fedora" +> > VERSION_ID="9.2" +> > [root@nine ~]# +> +> > I did the test without the above patch and the original problem is +> > reproduced. +> +> > > This is also used in perf_pending_irq() and on PREEMPT_RT this is +> > > invoked from softirq context which is preemptible. +> +> Humm, and then when going thru perf_pending_irq() we don't hit that +> scheduling on atomic. +> +> - Arnaldo diff --git a/a/content_digest b/N1/content_digest index 47a09af..b7cdfee 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -1,7 +1,6 @@ - "ref\0ZZcyzV8logh6BY0I@kernel.org\0" "From\0Arnaldo Carvalho de Melo <acme@kernel.org>\0" "Subject\0Re: 'perf test sigtrap' failing on PREEMPT_RT_FULL\0" - "Date\0Wed, 21 Feb 2024 16:26:26 -0300\0" + "Date\0Wed, 6 Mar 2024 10:06:30 -0300\0" "To\0Sebastian Andrzej Siewior <bigeasy@linutronix.de>\0" "Cc\0Mike Galbraith <efault@gmx.de>" Peter Zijlstra <peterz@infradead.org> @@ -15,148 +14,132 @@ " Artem Savkov <asavkov@redhat.com>\0" "\00:1\0" "b\0" - "In Thu, 4 Jan 2024 19:35:57 -0300, Arnaldo Carvalho de Melo wrote:\n" - "> Em Fri, Jul 28, 2023 at 05:07:18PM +0200, Sebastian Andrzej Siewior escreveu:\n" - "> > On 2023-07-26 08:10:45 [+0200], Mike Galbraith wrote:\n" - "> > > > [\302\240\302\240 52.848925] BUG: scheduling while atomic: perf/6549/0x00000002\n" - "\n" - "> > > Had bf9ad37dc8a not been reverted due to insufficient beauty, you could\n" - "> > > trivially make the sigtrap test a happy camper (wart tested in tip-rt).\n" + "> In Thu, 4 Jan 2024 19:35:57 -0300, Arnaldo Carvalho de Melo wrote:\n" + "> > +++ b/kernel/events/core.c\n" + "> > @@ -6801,7 +6801,7 @@ static void perf_pending_task(struct callback_head *head)\n" + "> > * If we 'fail' here, that's OK, it means recursion is already disabled\n" + "> > * and we won't recurse 'further'.\n" + "> > */\n" + "> >- preempt_disable_notrace();\n" + "> >+ migrate_disable();\n" + "> > rctx = perf_swevent_get_recursion_context();\n" " \n" - "> > Thank you for the pointer Mike.\n" + "> Pardon my ignorance, is it safe to call preempt_count() with preemption\n" + "> enabled on PREEMPT_RT, or at least in the context being discussed here?\n" " \n" - "> > I guess we need this preempt_disable_notrace() in perf_pending_task()\n" - "> > due to context accounting in get_recursion_context(). Would a\n" - "> > migrate_disable() be sufficient or could we send the signal outside of\n" - "> > the preempt-disabled block?\n" - "\n" - "> I got back to this, need to go again over all the callers of\n" - "> perf_swevent_get_recursion_context(), from the first quick glance there\n" - "> are other places with preempt_disable()/enable(), but doing just the\n" - "> switch to migrate disable/enable on perf_pending_task() makes this\n" - "> specific test to work:\n" - "\n" - "> [acme@nine linux]$ git log --oneline -5\n" - "> 086dab66d504 (HEAD -> linux-rt-devel/linux-6.7.y-rt/send_sig_perf.fix, tag: v6.7-rc5-rt5, linux-rt-devel/linux-6.7.y-rt) v6.7-rc5-rt5\n" - "> 29e0d951f39b printk: Update the printk series.\n" - "> 2308ecc8ce88 (tag: v6.7-rc5-rt4) v6.7-rc5-rt4\n" - "> 10d5f3551216 Merge tag 'v6.7-rc5' into linux-6.7.y-rt\n" - "> a39b6ac3781d (tag: v6.7-rc5, linux-rt-devel/master, linux-rt-devel/linux-6.7.y) Linux 6.7-rc5\n" - "> [acme@nine linux]$ git diff\n" - "> diff --git a/kernel/events/core.c b/kernel/events/core.c\n" - "> index c9d123e13b57..a9b9ef60f6b3 100644\n" - "> --- a/kernel/events/core.c\n" - "> +++ b/kernel/events/core.c\n" - "> @@ -6801,7 +6801,7 @@ static void perf_pending_task(struct callback_head *head)\n" - "> * If we 'fail' here, that's OK, it means recursion is already disabled\n" - "> * and we won't recurse 'further'.\n" - "> */\n" - ">- preempt_disable_notrace();\n" - ">+ migrate_disable();\n" - "> rctx = perf_swevent_get_recursion_context();\n" - "\n" - "Pardon my ignorance, is it safe to call preempt_count() with preemption\n" - "enabled on PREEMPT_RT, or at least in the context being discussed here?\n" - "\n" - "Because:\n" - "\n" - "\t perf_swevent_get_recursion_context()\n" - "\t get_recursion_context()\n" - " interrupt_context_level()\n" - " preempt_count()\t\n" - "\n" - "And:\n" - "\n" - "int perf_swevent_get_recursion_context(void)\n" - "{\n" - " struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);\n" - "\n" - " return get_recursion_context(swhash->recursion);\n" - "}\n" - "\n" - "> if (event->pending_work) {\n" - "> @@ -6812,7 +6812,7 @@ static void perf_pending_task(struct callback_head *head)\n" + "> Because:\n" " \n" - "> if (rctx >= 0)\n" - "> perf_swevent_put_recursion_context(rctx);\n" - "> - preempt_enable_notrace();\n" - "> + migrate_enable();\n" + "> \t perf_swevent_get_recursion_context()\n" + "> \t get_recursion_context()\n" + "> interrupt_context_level()\n" + "> preempt_count()\t\n" " \n" - "> put_event(event);\n" - "> }\n" - "> [acme@nine linux]$ uname -a\n" - "> Linux nine 6.7.0-rc5-rt5.sigtrap-fix-dirty #2 SMP PREEMPT_RT Thu Jan 4 18:11:44 -03 2024 x86_64 x86_64 x86_64 GNU/Linux\n" - "> [acme@nine linux]$ sudo su -\n" - "> [sudo] password for acme: \n" - "> [root@nine ~]# \n" - "> [root@nine ~]# perf test sigtrap\n" - "> 68: Sigtrap : Ok\n" - "> [root@nine ~]# \n" - "> [root@nine ~]# perf probe -L perf_pending_task\n" - "> <perf_pending_task@/home/acme/git/linux/kernel/events/core.c:0>\n" - "> 0 static void perf_pending_task(struct callback_head *head)\n" - "> {\n" - "> 2 struct perf_event *event = container_of(head, struct perf_event, pending_task);\n" - "> 3 int rctx;\n" - " \n" - "> /*\n" - "> * If we 'fail' here, that's OK, it means recursion is already disabled\n" - "> * and we won't recurse 'further'.\n" - "> */\n" - "> migrate_disable();\n" - "> 10 rctx = perf_swevent_get_recursion_context();\n" - "> \n" - "> 12 if (event->pending_work) {\n" - "> 13 event->pending_work = 0;\n" - "> 14 perf_sigtrap(event);\n" - "> 15 local_dec(&event->ctx->nr_pending);\n" - "> }\n" - "> \n" - "> 18 if (rctx >= 0)\n" - "> 19 perf_swevent_put_recursion_context(rctx);\n" - "> 20 migrate_enable();\n" - " \n" - "> 22 put_event(event);\n" - "> }\n" - " \n" - "> #ifdef CONFIG_GUEST_PERF_EVENTS\n" - "\n" - "> [root@nine ~]# perf probe perf_pending_task\n" - "> Added new event:\n" - "> probe:perf_pending_task (on perf_pending_task)\n" + "> And:\n" + " \n" + "> int perf_swevent_get_recursion_context(void)\n" + "> {\n" + "> struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);\n" + "> \n" + "> return get_recursion_context(swhash->recursion);\n" + "> }\n" "\n" - "> You can now use it in all perf tools, such as:\n" + "Seems to be enough because perf_pending_task is a irq_work callback and\n" + "that is guaranteed not to reentry?\n" "\n" - "> \tperf record -e probe:perf_pending_task -aR sleep 1\n" + "Artem's tests with a RHEL kernel seems to indicate that, ditto for my,\n" + "will test with upstream linux-6.8.y-rt.\n" "\n" - "> [root@nine ~]# perf trace --max-events=1 -e probe:perf_pending_task/max-stack=6/ perf test sigtrap \n" - "> 68: Sigtrap : Ok\n" - "> 0.000 :9608/9608 probe:perf_pending_task(__probe_ip: -2064408784)\n" - "> perf_pending_task ([kernel.kallsyms])\n" - "> task_work_run ([kernel.kallsyms])\n" - "> exit_to_user_mode_loop ([kernel.kallsyms])\n" - "> exit_to_user_mode_prepare ([kernel.kallsyms])\n" - "> irqentry_exit_to_user_mode ([kernel.kallsyms])\n" - "> asm_sysvec_irq_work ([kernel.kallsyms])\n" - "> [root@nine ~]#\n" + "But there is a lot more happening in perf_sigtrap and I'm not sure if\n" + "the irq_work callback gets preempted we would not race with something\n" + "else.\n" "\n" - "> [root@nine ~]# head -5 /etc/os-release\n" - "> NAME=\"Red Hat Enterprise Linux\"\n" - "> VERSION=\"9.2 (Plow)\"\n" - "> ID=\"rhel\"\n" - "> ID_LIKE=\"fedora\"\n" - "> VERSION_ID=\"9.2\"\n" - "> [root@nine ~]#\n" + "Marco, Mike, ideas?\n" "\n" - "> I did the test without the above patch and the original problem is\n" - "> reproduced.\n" + "- Arnaldo\n" " \n" - "> > This is also used in perf_pending_irq() and on PREEMPT_RT this is\n" - "> > invoked from softirq context which is preemptible.\n" - "\n" - "Humm, and then when going thru perf_pending_irq() we don't hit that\n" - "scheduling on atomic.\n" - "\n" - - Arnaldo + "> > if (event->pending_work) {\n" + "> > @@ -6812,7 +6812,7 @@ static void perf_pending_task(struct callback_head *head)\n" + "> \n" + "> > if (rctx >= 0)\n" + "> > perf_swevent_put_recursion_context(rctx);\n" + "> > - preempt_enable_notrace();\n" + "> > + migrate_enable();\n" + "> \n" + "> > put_event(event);\n" + "> > }\n" + "> > [acme@nine linux]$ uname -a\n" + "> > Linux nine 6.7.0-rc5-rt5.sigtrap-fix-dirty #2 SMP PREEMPT_RT Thu Jan 4 18:11:44 -03 2024 x86_64 x86_64 x86_64 GNU/Linux\n" + "> > [acme@nine linux]$ sudo su -\n" + "> > [sudo] password for acme: \n" + "> > [root@nine ~]# \n" + "> > [root@nine ~]# perf test sigtrap\n" + "> > 68: Sigtrap : Ok\n" + "> > [root@nine ~]# \n" + "> > [root@nine ~]# perf probe -L perf_pending_task\n" + "> > <perf_pending_task@/home/acme/git/linux/kernel/events/core.c:0>\n" + "> > 0 static void perf_pending_task(struct callback_head *head)\n" + "> > {\n" + "> > 2 struct perf_event *event = container_of(head, struct perf_event, pending_task);\n" + "> > 3 int rctx;\n" + "> \n" + "> > /*\n" + "> > * If we 'fail' here, that's OK, it means recursion is already disabled\n" + "> > * and we won't recurse 'further'.\n" + "> > */\n" + "> > migrate_disable();\n" + "> > 10 rctx = perf_swevent_get_recursion_context();\n" + "> > \n" + "> > 12 if (event->pending_work) {\n" + "> > 13 event->pending_work = 0;\n" + "> > 14 perf_sigtrap(event);\n" + "> > 15 local_dec(&event->ctx->nr_pending);\n" + "> > }\n" + "> > \n" + "> > 18 if (rctx >= 0)\n" + "> > 19 perf_swevent_put_recursion_context(rctx);\n" + "> > 20 migrate_enable();\n" + "> \n" + "> > 22 put_event(event);\n" + "> > }\n" + "> \n" + "> > #ifdef CONFIG_GUEST_PERF_EVENTS\n" + "> \n" + "> > [root@nine ~]# perf probe perf_pending_task\n" + "> > Added new event:\n" + "> > probe:perf_pending_task (on perf_pending_task)\n" + "> \n" + "> > You can now use it in all perf tools, such as:\n" + "> \n" + "> > \tperf record -e probe:perf_pending_task -aR sleep 1\n" + "> \n" + "> > [root@nine ~]# perf trace --max-events=1 -e probe:perf_pending_task/max-stack=6/ perf test sigtrap \n" + "> > 68: Sigtrap : Ok\n" + "> > 0.000 :9608/9608 probe:perf_pending_task(__probe_ip: -2064408784)\n" + "> > perf_pending_task ([kernel.kallsyms])\n" + "> > task_work_run ([kernel.kallsyms])\n" + "> > exit_to_user_mode_loop ([kernel.kallsyms])\n" + "> > exit_to_user_mode_prepare ([kernel.kallsyms])\n" + "> > irqentry_exit_to_user_mode ([kernel.kallsyms])\n" + "> > asm_sysvec_irq_work ([kernel.kallsyms])\n" + "> > [root@nine ~]#\n" + "> \n" + "> > [root@nine ~]# head -5 /etc/os-release\n" + "> > NAME=\"Red Hat Enterprise Linux\"\n" + "> > VERSION=\"9.2 (Plow)\"\n" + "> > ID=\"rhel\"\n" + "> > ID_LIKE=\"fedora\"\n" + "> > VERSION_ID=\"9.2\"\n" + "> > [root@nine ~]#\n" + "> \n" + "> > I did the test without the above patch and the original problem is\n" + "> > reproduced.\n" + "> \n" + "> > > This is also used in perf_pending_irq() and on PREEMPT_RT this is\n" + "> > > invoked from softirq context which is preemptible.\n" + "> \n" + "> Humm, and then when going thru perf_pending_irq() we don't hit that\n" + "> scheduling on atomic.\n" + "> \n" + > - Arnaldo -146adbea900da2fe9b5476626383661e8a97a4111f05cd5c68d99cb9892a6058 +b9c48cdd55938b11f3dd12aab21a81b834776ebd579e600628dc4802f027780a
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).