[Fuego] [PATCH 8/9] signaltest: update to new framework

Wed Nov 8 04:06:24 UTC 2017

> -----Original Message-----
> From: Daniel Sangorrin on Monday, November 06, 2017 11:21 PM
> signaltest, like cyclictest, one testcase inside the rt-tests
> package. The output format is the same as cyclictest so I just
> copied the parser and criteria files.

OK. Everything looks OK.  I haven't had time to test this on my 
machines, but I applied and pushed to master.  Please test
when you get a chance.

> 
> TODO: there are more rt-tests (e.g. Functional.pi_tests) inside
> Fuego that need to be updated.
Agreed.  Should I make an issue for this?

> 
> TODO: some of the rt-tests are supposed to run in parallel.
> For example, cyclictest may run with hackbench in background.
> Unfortunately, Fuego currently executes tests on the board
> in a sequential manner. We need to make sure that Fuego can
> execute multiple jobs on the same board in parallel or at least
> provide functionality to "disturb" a test with some background
> load.
Agreed.  Should I make an issue for this also?
  -- Tim

> 
> Signed-off-by: Daniel Sangorrin <daniel.sangorrin at toshiba.co.jp>
> ---
>  .../tests/Benchmark.signaltest/chart_config.json   |   4 ++-
>  engine/tests/Benchmark.signaltest/criteria.json    |  26 +++++++++++++++
>  engine/tests/Benchmark.signaltest/fuego_test.sh    |  12 ++++---
>  engine/tests/Benchmark.signaltest/parser.py        |  35 ++++++++++----------
> -
>  engine/tests/Benchmark.signaltest/reference.log    |  17 ----------
>  .../tests/Benchmark.signaltest/signaltest.tar.gz   | Bin 3710 -> 0 bytes
>  engine/tests/Benchmark.signaltest/spec.json        |   3 +-
>  7 files changed, 53 insertions(+), 44 deletions(-)
>  create mode 100644 engine/tests/Benchmark.signaltest/criteria.json
>  delete mode 100644 engine/tests/Benchmark.signaltest/reference.log
>  delete mode 100644 engine/tests/Benchmark.signaltest/signaltest.tar.gz
> 
> diff --git a/engine/tests/Benchmark.signaltest/chart_config.json
> b/engine/tests/Benchmark.signaltest/chart_config.json
> index f8dcb3b..cdaf6a2 100644
> --- a/engine/tests/Benchmark.signaltest/chart_config.json
> +++ b/engine/tests/Benchmark.signaltest/chart_config.json
> @@ -1,3 +1,5 @@
>  {
> -        "signaltest":["Thread0", "Thread1"]
> +	"chart_type": "measure_plot",
> +	"measures": ["default.latencies.max_latency",
> +        "default.latencies.avg_latency"]
>  }
> diff --git a/engine/tests/Benchmark.signaltest/criteria.json
> b/engine/tests/Benchmark.signaltest/criteria.json
> new file mode 100644
> index 0000000..f137d55
> --- /dev/null
> +++ b/engine/tests/Benchmark.signaltest/criteria.json
> @@ -0,0 +1,26 @@
> +{
> +    "schema_version":"1.0",
> +    "criteria":[
> +        {
> +            "tguid":"default.latencies.max_latency",
> +            "reference":{
> +                "value":1000,
> +                "operator":"le"
> +            }
> +        },
> +        {
> +            "tguid":"default.latencies.min_latency",
> +            "reference":{
> +                "value":1000,
> +                "operator":"le"
> +            }
> +        },
> +        {
> +            "tguid":"default.latencies.avg_latency",
> +            "reference":{
> +                "value":1000,
> +                "operator":"le"
> +            }
> +        }
> +    ]
> +}
> diff --git a/engine/tests/Benchmark.signaltest/fuego_test.sh
> b/engine/tests/Benchmark.signaltest/fuego_test.sh
> index af62c0c..84b5063 100755
> --- a/engine/tests/Benchmark.signaltest/fuego_test.sh
> +++ b/engine/tests/Benchmark.signaltest/fuego_test.sh
> @@ -1,19 +1,21 @@
> -tarball=signaltest.tar.gz
> +gitrepo="https://git.kernel.org/pub/scm/utils/rt-tests/rt-tests.git"
> +gitref="unstable/devel/v1.1.1"
> 
>  NEED_ROOT=1
> 
>  function test_build {
> -  make CC="$CC" LD="$LD" LDFLAGS="$LDFLAGS" CFLAGS="$CFLAGS"
> +    patch -p1 -N -s < $TEST_HOME/../Benchmark.cyclictest/0001-Add-
> scheduling-policies-for-old-kernels.patch
> +    make NUMA=0 signaltest
>  }
> 
>  function test_deploy {
> -	put signaltest  $BOARD_TESTDIR/fuego.$TESTDIR/
> +    put signaltest  $BOARD_TESTDIR/fuego.$TESTDIR/
>  }
> 
>  function test_run {
> -	assert_define BENCHMARK_SIGNALTEST_LOOPS
> +    assert_define BENCHMARK_SIGNALTEST_PARAMS
> 
> -	report "cd $BOARD_TESTDIR/fuego.$TESTDIR; ./signaltest  -l
> $BENCHMARK_SIGNALTEST_LOOPS -q"
> +    report "cd $BOARD_TESTDIR/fuego.$TESTDIR; ./signaltest
> $BENCHMARK_SIGNALTEST_PARAMS"
>  }
> 
> 
> diff --git a/engine/tests/Benchmark.signaltest/parser.py
> b/engine/tests/Benchmark.signaltest/parser.py
> index 44425ea..3f4f05d 100755
> --- a/engine/tests/Benchmark.signaltest/parser.py
> +++ b/engine/tests/Benchmark.signaltest/parser.py
> @@ -1,26 +1,23 @@
>  #!/usr/bin/python
> -# See common.py for description of command-line arguments
> -
>  import os, re, sys
>  sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser')
>  import common as plib
> 
> -ref_section_pat = "^\[[\w\d_ .]+.[gle]{2}\]"
> -cur_search_pat =
> re.compile("^T:([\s\d]+)(.*)P:(.*)C:(.*)Min:([\s\d]+)Act:([\s\d]+)Avg:([\s\d]
> +)Max:([\s\d]+)",re.MULTILINE)
> -
> -res_dict = {}
> -cur_dict = {}
> -
> -pat_result = plib.parse(cur_search_pat)
> -if pat_result:
> -	cur_dict["Thread0.Min"] = '%d' % int(pat_result[0][4])
> -	cur_dict["Thread0.Act"] = '%d' % int(pat_result[0][5])
> -	cur_dict["Thread0.Avg"] = '%d' % int(pat_result[0][6])
> -	cur_dict["Thread0.Max"] = '%d' % int(pat_result[0][7])
> -	cur_dict["Thread1.Min"] = '%d' % int(pat_result[1][4])
> -	cur_dict["Thread1.Act"] = '%d' % int(pat_result[1][5])
> -	cur_dict["Thread1.Avg"] = '%d' % int(pat_result[1][6])
> -	cur_dict["Thread1.Max"] = '%d' % int(pat_result[1][7])
> +regex_string = "^T:.*Min:\s+(\d+).*Avg:\s+(\d+) Max:\s+(\d+)"
> +measurements = {}
> +matches = plib.parse_log(regex_string)
> 
> +if matches:
> +	min_latencies = []
> +	avg_latencies = []
> +	max_latencies = []
> +	for thread in matches:
> +		min_latencies.append(float(thread[0]))
> +		avg_latencies.append(float(thread[1]))
> +		max_latencies.append(float(thread[2]))
> +	measurements['default.latencies'] = [
> +		{"name": "max_latency", "measure" : max(max_latencies)},
> +		{"name": "min_latency", "measure" : min(min_latencies)},
> +		{"name": "avg_latency", "measure" :
> sum(avg_latencies)/len(avg_latencies)}]
> 
> -sys.exit(plib.process_data(ref_section_pat, cur_dict, 'm', 'usec'))
> +sys.exit(plib.process(measurements))
> diff --git a/engine/tests/Benchmark.signaltest/reference.log
> b/engine/tests/Benchmark.signaltest/reference.log
> deleted file mode 100644
> index 83a959d..0000000
> --- a/engine/tests/Benchmark.signaltest/reference.log
> +++ /dev/null
> @@ -1,17 +0,0 @@
> -#sdfdsf
> -[Thread0.Min|le]
> -10000000
> -[Thread0.Act|le]
> -10000000
> -[Thread0.Avg|le]
> -10000000
> -[Thread0.Max|le]
> -10000000
> -[Thread1.Min|le]
> -10000000
> -[Thread1.Act|le]
> -10000000
> -[Thread1.Avg|le]
> -10000000
> -[Thread1.Max|le]
> -10000000
> diff --git a/engine/tests/Benchmark.signaltest/signaltest.tar.gz
> b/engine/tests/Benchmark.signaltest/signaltest.tar.gz
> deleted file mode 100644
> index
> 2c15b87f03a88b2d2818994223be08837d89e343..0000000000000000000000000
> 000000000000000
> GIT binary patch
> literal 0
> HcmV?d00001
> 
> literal 3710
> zcmV-^4uSC>iwFqm`FK(Q19NF-
> ZeeV6Wpi{cbYXG;?HX%y+cq*^rC+gj&o1Riwj{?+
> zDo4(xu9JGskBsf~rsw83GzCelDN-RpKic|!`|U0O5|m^onQ41-cf at pJaIsh{7W-
> Dy
> zU=(?w6sg>J@<)v}j^|rj^tbbTt8M=^<4LF6-
> Q4bMZfz30v$_3z>j`ZAVH{Z_OQn|p
> zJn_Rg8;pfFcmVHH>;I)AE#CjgyAs176u<i!^uXKCo;~>f-RGO;{kz?*O?>~Y&CTs6
> z(Ei;ySup<3-+ylpUiING&fe+C*_pd{a`f)taKAC~{rdUq)3^KQ(1$`r>TeETzdb`x
> z>+fC|LhGbkKRo!B0pV1R6O<0l5?Qa~a?jeh?$>L0#$E$YVj4L4C$wVxRL9T)k!sZ
> I
> zeki=CSFa@#Xbmldwe=X*2VyFs0nXR{RsT<m?tfxf_HRGe?|R1L_Kzmtc|K$R+
> uiQ7
> z%Krb1v9VT%H8?%j#tKQCMFW`xQy at ivG#<(uFA<1Dcn(@{7lS#RkK>7#!rM>;
> w~<KT
> zr5uH~e+`2ulSv!}w`i!lde8YdNMV}9qr{s4PKSvQIo%z&i!<=O2of;}Qat!FlL7(>
> zUNqQ<6CyB)2f^@;V8|MtCC37pOC~AAL;iSs{2ty4WCQtsmtBT|4~Ky-
> qEx^SA_?L&
> zA~Ib-
> hNrZgj>P~j?|APU;^s_q18?Z|o($q>M{&PVuYWbbs2Cxs{nN99lVkVn{Pf`X
> zEiC^4?e)!NTn{2Y%mxBp4*f`m>*HMuG!inN$}&uQ$}&)ie+DK8VQ at JI3*uQ&
> 5=5gi
> zOa>EChGY?nmk+go^kBCe3zyQnbYq(Mvlk*TT8x~B%DbsZ7l1V<xg1}gp1%IkJ
> v;cv
> zzH{3ECk=iP at sOq)py&8;(f-
> iDSg^4M??>TX6HqH)9S6OLA%wD(!N6$%9^+%caa{NK
> z)K$<%ao+p0{XO^H{;7+9YqgGYI{E7HqE$kmvyad at z@>m`e1m$2+SY|7n}Z)-
> AAbA(
> z4c`U#Zg0ML#@o(hlKB#33^MVebclPW9%{cptxXwB8mz)4QcE>D16;<0U+c9X
> k`N4b
> z)MScktfxv-
> @FR<8imDvO(Fi8qt?S=m at Jw;3Op at LU*680<)40rrlk{q*e!pWgJd6_{
> zEYya;Pv==<I={{yoFBRfO#5(2aTrAT)VnoP&zCqI#AuiTyz7yHuU&g##&os4xQ3
> 51
> zD9(X&G4(+%U5Xq=Qly*m-F#pnjx#xkZ;&Aih_B^~FqojH=|j7;G~~58a2yV-
> g<>Tn
> z?YNGE0mw9plZj&liN=slg%4{oZPZbeH)Bk=!I5ceS6;j5v3%I=mz1(W60g|^tun$
> +
> zv5c!vpW^p@{eG4i^(J2Eqc0uKu_HT8n1gkhL*7W6=#}sfUe&fJ3JMO2qJcEl6${
> V;
> z4e4q~!|Z+9FJnjZ=ma8PK&FV>5+SoB(hS^_wvsQ`!AiZh at zgBuHJ!!qilabmQhb
> ?3
> zrbf$nDgG!-C-pH%XbhfDX)PF~Lb_~&II#@P2<=j!5s4;vL$vir3{f4}+O>^x8e`<T
> zGb?2A at CwfMzTf}OeRJ^U1bX`8<ox^nQ{xOUyiC#LSpOI at 43d<{cTgW|!4P#T
> *J`Co
> z=q;s~ILpEx$IyXYKsO)z8|166L=uT`gLk;-
> QC)wUrOWhspS_Esdy+*FMx+HB+YfKV
> z`$p3Br0XYMI{valvs5H5M#0fYU>dPtZ|gHRyFv2t%h<ja$z`01&)qad at l2+uLZ}c6
> zZ at F;S4OZsa)I}}(0<oH-
> C at _Sc6p??YwvtjPf7ZaS&k*v&$FxYgBJ$8jvv|MYZLD4B
> zm&hVKiOwn at oi!1h2%<?5yLV_;&Z at GnhB7&CFtC74Z8&V^8iP*8bo9zOJ9z7
> Udw8<<
> zw<hH1$L|jh`Gkp>C^~G;O=vew!l57yrpD8CZT at J3Ngzu$X>ic5NuFe(NF1c4%$
> E*U
> z)OL!?+ChmN^kTy#m<cHSGAy16GC1(=@C4gy{}Q85Uu$&uK}DDf!sm32Sm_v
> HW6|PY
> zxtfl%N0M=mPL5B`PmT}vkX7Zbm{lE#mP};S&SJ*DMez-
> tXa at z8l%%W{bP4>p at dE3`
> zc+!BE&{p?~F$an>$s0b!B70J$a_IYOHT3=mLn;Qal%PCH`8y?Ie9%ycsq(%c&^G
> U&
> zv#mC9VFOgzr0WdX(A+NZ)1p4EuELU|UQ8t~unO(l_L~MXoiS;LGepA<Rvi-
> jEJ;M9
> zWTdEm>Pjd?em9fFTm_iCYE_BNO;DBJpPhE{qt33VCd)6(92S>gpn9^Ji2xK4YBl
> x5
> z_^F?+=Esr5v|rE%>4=Hyd2+zgYeKFY^EBv0-t`g`+vt~+FEL+;o2!RG-ZjYHt&Qy6
> zTF459X&|QxDl2U0Rr6jZ=HYu;85 at 3?V=TkcBGFqoIa*+7vzFH8TjzFF7?xfzHlpa
> =
> z7kvA2?k-A|l-2XetZpg+x5S{b-HRFZvv5D;lx;di)=vVM1 at -
> %26XfGAQN6B|dq5NH
> z)k(L~n?U+ws!S+Z#l(vA>LE|qe$WG2>B%SEAoUD6MbQZ!d1+4Ys2+Joxd3FTH
> xih?
> z4v8nK$yG(dSuH1agCL!T-
> km!Z;gnN7&ha>r7&^;EI}29QpQ7dFOa)I*X>i}uQ9VSe
> z*FX(>`%5h0@%VK2r>KO|y5v+Cpw(ijvO at J4lv0<Hfl3>}k0%okVK-
> wDfey#0s)Sso
> zIfRGCFpGQ^gQDm#DdYhiAu3*QEgEG?Asn8ZygNhg!Z at C$ef32n$|jeppyibw
> U_I+)
> zq3pLE<Qp|5ogp=acc%v at bWu`-
> zWU&iD+Ur92ct2`i&Izi+GR2y^{mzUm<8xMi@^^8
> zaTMNx2Wf_~Lf4H~6}N#b(~zhtM@&QOQ(t}Cx6&)cy}D%@*Et_LearbklbB`_l
> r<6q
> zDH?h<TN;<BJ@{S#N^^S<_59v7M16 at l3UUd4pED}B?(;#Y<EZRNlvI*#?2w<iF
> BPmt
> z&ZY7bX}S8YXiweJn%xhu!1}@^7(d32w!GNK=QrxgrK8sVS?ilBRM#w15eHr}@
> |)m~
> zv7}nV`SryIwn<tNNfIa8e~>txogJO`(91IsH;4w})`YH&wKX`=#A$=-rKy;w8PVy9
> z+RZhtVdsMUCHwwgmkZ-
> WCy)rdE>;OHvX at OTaH|Zl3144+ecx0Yh6Xu`=h22a8XM1}
> zB*Prnyx*hf(;S_j<N*dg=2-geAEhaCY*VT%rh1*@kvn|=>6JlJ-)*zdrWkl{l0jdk
> z_V^MmCyI{J3etxrEMNA9VQ<>|D0|n-
> Ow4{Zj2pHp<n at YAv8>f8MX;I<TaK^v$oJ5Y
> z at fCaJMvf=rz(JyTd}9al%?dk2Y&&2+jy*+eD-u46`w?-Km!n#Ic2(kP*?J!(c8%!9
> zRV^a-
> 3fGN&*{k=e4B&LZ at QqnG{PtUMk6rlY8<K0WU+=K_dkNizI1FfT5?RDza7ylr
> zy#*!%m=mkYu!z-Ef4LGuL7hSN8hFc#?$-7P)6Kw4c;fxsq~;-J_t_;KJf!R)o{Gp>
> zRz)XnN>OJ}X;0kK?@o>ne?*ns-rgqlLE4DT-5?@%O{r~nYm1gJPfrCxX-?-
> NYO6zj
> z1G9cga#G)mh7RvoKBP`L=!0Nou+IPCiL1=sD)YCB3=%Pt41!5#yOfu4LfnUKnj;E
> I
> zv-
> 5(4X0;DHa}vV at WQ;D@(dCtrmrgtvZcrT*f;bQLxti?n*|eH=I9Sjed$3ofuD4`%
> z=B2Ar_9f&@C*3BwI`6G?2jHx14I1!{=X<@C=V1oRgk{3$Q4kSy0Iz-7TiK$?^+-
> *Q
> zyxZQ&i;$9urnN<1BX1})iz3}dvoqjkQLmUOrcMuG!z*jSAve`<@&aX}lvzWM+X
> ;Dn
> zV~h`|hG22%`np7ashppa+_miT0vim<f&zhmKqwapJGS6gUJOWXSuVV?$O<A
> &K1$a%
> zGi!N`!%bzCSp?Bsu;*y6ZF;szYJRoA%%^Ya+PU6np`?$rO*OHA0-
> nrE2+}@`HzK26
> zk`ka1K_sx$z`7>MBKr9+iDS8>6D^f#wenucOg>`;Ev{io4(tm%R_gLI%_?}jTczp`
> z!Z|oTZyIl)Y3P;n{nI0R2`|CDZ2C1U_H74Vw7TYn-
> Y_nUIw9niivCPZ+y7 at 1B>aLN
> ze^~_iadq|=aGD1JIzx*r0-ioCd`ij5Ff$jyhjpEu=WJO&!{uzTwKno{f2q7zSN)nT
> zq at wJ`O9(}o5|X#*LW>-(ZkT91Zea#H1%T_GzLlzfg-
> 5qdxvrrNZK3?pY9SJqT*0uQ
> zqJP`yYpt>9mlBoI?|u1Ra?!ZBs;_I>5WJhaGWr+J(gJs2S_ at XF4}-
> 6?UINaR at g2LB
> z<>S9nIICr3lP21~&7jUJrj7kAZa}i13B%HH2>v`3Qr*afe532_DlI5cZ<~e=D^0P~
> zD0j{t=YW-kxv;g?wDz<|zX+ao3{xfX&Y-nzq at sB<-
> #jcZEr;`vSlMZZ_`W9D%Cjv@
> zf|^R=)-
> Ov&ASMT;Rw>7O({h~6$imngAoYbAS#35aTO~&FtgKeb%Mh8koJq~B7?&31
> zHn(S*e62=dtVLGD3fKO?b&K&Vs=#o{eKaFZ4cWD=jo-
> A8k8bSEbceO(nQRTy?P5o{
> zhIv4&_7#6>Z*E?!3|`~aHkGl|RP;khX!q5dO$`l0TZW-LLnG3rF6r)ki8EeoNV~eJ
> zKd+CL`RiC!NM;c!lQwEXKgXD6Eb=3TS1)8W_sju4b6^^__cO;@L4ba*oP{eH=P
> L~*
> zW~AEFP>PcZ*wYQ0gk%4G_5UPRqZ-
> wyMm4HYjcQb*8r7&qHL6jKYE+{d)u=`_s!@$<
> cRHGWzs75uaQH^R;<If%c26d at S;s8(p0Jbtg`Tzg`
> 
> diff --git a/engine/tests/Benchmark.signaltest/spec.json
> b/engine/tests/Benchmark.signaltest/spec.json
> index 4cddb88..01b0f88 100644
> --- a/engine/tests/Benchmark.signaltest/spec.json
> +++ b/engine/tests/Benchmark.signaltest/spec.json
> @@ -2,8 +2,7 @@
>      "testName": "Benchmark.signaltest",
>      "specs": {
>          "default": {
> -            "LOOPS":"10000"
> +            "PARAMS": "-t 2 -m -l 10000 -p 60 -q"
>          }
>      }
>  }
> -
> --
> 2.7.4
> 
> 
> _______________________________________________
> Fuego mailing list
> Fuego at lists.linuxfoundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/fuego