From f1a0cedd3a0b170cbf04589348ef20741014e97c Mon Sep 17 00:00:00 2001 From: Harald Welte Date: Thu, 12 Apr 2018 15:33:48 +0200 Subject: osmodevcon2018 osmo-ttcn3 testsuites --- .../osmodevcon2018-ttcn3_test_suites.adoc | 225 + .../osmodevcon2018-ttcn3_test_suites.html | 4503 ++++++++++++++++++++ .../osmodevcon2018-ttcn3_test_suites__1.png | Bin 0 -> 37889 bytes .../osmodevcon2018-ttcn3_test_suites__2.png | Bin 0 -> 44485 bytes .../osmodevcon2018-ttcn3_test_suites__3.png | Bin 0 -> 34802 bytes 5 files changed, 4728 insertions(+) create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.html create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__1.png create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__2.png create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__3.png diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc new file mode 100644 index 0000000..ae02b96 --- /dev/null +++ b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc @@ -0,0 +1,225 @@ +Osmocom TTCN-3 Test Suites +========================== +:author: Harald Welte +:copyright: 2018 by Harald Welte (License: CC-BY-SA) +:backend: slidy +:max-width: 45em + + +== Osmocom TTCN-3 Test Suites + +* developed in 2017+2018 +* compiled using Eclipse TITAN +** uses just a command-line compiler + Makefiles +** no IDE needed at all, don't let _Eclipse_ fool you +* containerized in Docker +* executed by Jenkins CI + +== Terminology + +ATS:: Abstract Test Suite +MTC:: Main Test Component +PTC:: Parallel Test Component +IUT:: Implementation Under Test + +== Test Suite Philosophy + +* test one network element (our IUT) +* test external behavior (3GPP and non-3GPP) +* emulate entire environment from TTCN-3 +* don't reuse Osmocom C-code protocol implementations in the tests +* test against independent TTCN-3 implementations! + +== What to test? + +* successful cases +* erroneous cases (no answer, NACK, ...) +** many difficult to reproduce with real phones/devices +* load / resource exhaustion +* spec compliance +* focus on functionality actually relevant to IUT + +== Why TTCN-3 + TITAN + +* TTCN-3 specifically designed for telecom protocol testing +* TITAN team released many telecom protocols in TTCN-3, such as +** BSSAP, L3 (RR/MM/CC), SMS (CP/RP/TP), SS, M3UA, SCCP, GTP, NS, BSSGP, ... +** shortens our test development cycle +** permits us to test against known working industry implementations + +== Test suites for Osmocom CNI components + +* `osmo-bts` +* `osmo-bsc` +* `osmo-msc` +* `osmo-mgw` +* `osmo-hlr` +* `osmo-sip-connector` +* `osmo-sgsn` +* `osmo-ggsn` + +== Test suites in progress + +* `osmo-pcu` +* `osmo-bsc_nat` + + + + +== BTS_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BSC-side server) +** Um side: L1CTL to control MS +** PCU side: pcu_socket + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS, BSC}; + BTS [label="IUT\nosmo-bts-trx",shape="box"]; + ATS [label="ATS\nBTS_Tests.ttcn"]; + BSC [label="osmo-bsc\nOML only"]; + BTS -> fake_trx [label="bursts"]; + fake_trx -> trxcon [label="bursts"]; + trxcon -> ATS [label="bursts"]; + + BTS -> BSC [label="A-bis OML"]; + BTS -> ATS [label="A-bis RSL"]; + + ATS -> BTS [label="pcu_sock"]; + ATS -> BSC [label="VTY"]; + ATS -> BTS [label="CTRL"]; +} +---- + + + +== BSC_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BTS-side client) +** A-side: BSSAP/SCCP/M3UA (emulates MSC-side) +** MGW side: MGCP (emulates MGW side) + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS; STP; }; + BSC [label="IUT\nosmo-bsc",shape="box"]; + ATS [label="ATS\nBSC_Tests.ttcn"]; + BTS [label="osmo-bts-omldummy\nOML only"]; + + BTS -> BSC [label="A-bis OML"]; + ATS -> BSC [label="A-bis RSL"]; + ATS -> BSC [label="CTRL"]; + ATS -> BSC [label="VTY"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + BSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + +== MSC_Tests.ttcn + +* external interfaces +** A: BSSAP/SCCP/M3UA (emulates BSC-side) +** MNCC: MNCC/unix-domain (emulates ext. MNCC side) +** MGW: MGCP (emulates MGW side) +** GSUP (impllements HLR side) + +[graphviz] +---- +digraph G { + rankdir=LR; + MSC [label="IUT\nosmo-msc",shape="box"]; + ATS [label="ATS\nMSC_Tests.ttcn"]; + + ATS -> MSC [label="MNCC"]; + ATS -> MSC [label="SMPP",style="dashed"]; + ATS -> MSC [label="CTRL"]; + ATS -> MSC [label="VTY"]; + MSC -> ATS [label="GSUP"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + MSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + + +== MGCP_Test.ttcn + +* external interfaces +** MGCP (emulates call agent) +** RTP (stream source/sink) + +== HLR_Tests.ttcn + +* external interfaces +** GSUP (emulates VLR/SGSN side) + +== SIP_Tests.ttcn + +* external interfaces +** MNCC (emulates MSC side) +** SIP (emulates SIP switch) + +== SGSN_Tests.ttcn + +* external interfaces +** Gb (emulates PCU side NS/BSSGP + MS) +** GSUP (emulates HLR) + +== GGSN_Tests.ttcn + +* external interfaces +** Gp: GTP (emulates SGSN) +** Gi: IP (emulates Internet) + + +== Dockerized Setup + +* one process per container +* packages either +** IUT (e.g. `osmo-bsc`) +** ATS (compiled docker test suite) +** other utility (e.g. `trxcon` or `osmo-bts-omldummy`) +* why? +** no need for local ip/network configuration +** standardized / packaged setup on every machine +** run older/newer versions of ATS against older/newer IUT + +== Jenkins CI Execution + +. update `docker-playground.git` +.. contains `Dockerfile` for ATS + IUT +. rebuild IUT container[s] (e.g. `osmo-bts-master`) +.. git magic ensures re-build only if `osmo-bts.git` master changed +. rebuild ATS container (e.g. `ttcn3-bts-test`) +.. git magic ensures re-build only if `osmo-ttcn3-hacks.git` master changed +. run `docker-playground/ttcn3-bts-test/jenkins.sh` +.. creates docker network +.. starts IUT + ATS docker containers +.. collects test results + + +== Jenkins CI Reporting + +* junit-xml generation +* store artefacts +** pcap file of every test case +** ATS log file (TTCN-3 testsuite) +** IUT log file[s] (`osmo-*.log`) +** IUT config file[s] (`osmo-*.cfg`) +* see https://jenkins.osmocom.org/jenkins/view/TTCN3/ + + +== Further Reading + +* http://git.osmocom.org/osmo-ttcn3-hacks/ +* http://git.osmocom.org/docker-playground/ +* http://osmocom.org/projects/cellular-infrastructure/wiki/Titan_TTCN3_Notes + +== EOF + +End of File diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.html b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.html new file mode 100644 index 0000000..d4947c7 --- /dev/null +++ b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.html @@ -0,0 +1,4503 @@ + + + + +Osmocom TTCN-3 Test Suites + + + + + + + + +
+

Osmocom TTCN-3 Test Suites

+
+
    +
  • + +developed in 2017+2018 + +
    • +
  • + +compiled using Eclipse TITAN + +
      +
    • + +uses just a command-line compiler + Makefiles + +
      • +
    • + +no IDE needed at all, don’t let Eclipse fool you + +
      • +
    +
    • +
  • + +containerized in Docker + +
    • +
  • + +executed by Jenkins CI + +
    • +
+
+
+
+

Terminology

+
+
+
+ATS +
+
+

+Abstract Test Suite +

+
+
+MTC +
+
+

+Main Test Component +

+
+
+PTC +
+
+

+Parallel Test Component +

+
+
+IUT +
+
+

+Implementation Under Test +

+
+
+
+
+
+

Test Suite Philosophy

+
+
    +
  • + +test one network element (our IUT) + +
    • +
  • + +test external behavior (3GPP and non-3GPP) + +
    • +
  • + +emulate entire environment from TTCN-3 + +
    • +
  • + +don’t reuse Osmocom C-code protocol implementations in the tests + +
    • +
  • + +test against independent TTCN-3 implementations! + +
    • +
+
+
+
+

What to test?

+
+
    +
  • + +successful cases + +
    • +
  • + +erroneous cases (no answer, NACK, …) + +
      +
    • + +many difficult to reproduce with real phones/devices + +
      • +
    +
    • +
  • + +load / resource exhaustion + +
    • +
  • + +spec compliance + +
    • +
  • + +focus on functionality actually relevant to IUT + +
    • +
+
+
+
+

Why TTCN-3 + TITAN

+
+
    +
  • + +TTCN-3 specifically designed for telecom protocol testing + +
    • +
  • + +TITAN team released many telecom protocols in TTCN-3, such as + +
      +
    • + +BSSAP, L3 (RR/MM/CC), SMS (CP/RP/TP), SS, M3UA, SCCP, GTP, NS, BSSGP, … + +
      • +
    • + +shortens our test development cycle + +
      • +
    • + +permits us to test against known working industry implementations + +
      • +
    +
    • +
+
+
+
+

Test suites for Osmocom CNI components

+
+
    +
  • + +osmo-bts + +
    • +
  • + +osmo-bsc + +
    • +
  • + +osmo-msc + +
    • +
  • + +osmo-mgw + +
    • +
  • + +osmo-hlr + +
    • +
  • + +osmo-sip-connector + +
    • +
  • + +osmo-sgsn + +
    • +
  • + +osmo-ggsn + +
    • +
+
+
+
+

Test suites in progress

+
+
    +
  • + +osmo-pcu + +
    • +
  • + +osmo-bsc_nat + +
    • +
+
+
+
+

BTS_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +A-bis side: RSL (emulates BSC-side server) + +
      • +
    • + +Um side: L1CTL to control MS + +
      • +
    • + +PCU side: pcu_socket + +
      • +
    +
    • +
+
+
+osmodevcon2018-ttcn3_test_suites__1.png +
+
+
+
+
+

BSC_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +A-bis side: RSL (emulates BTS-side client) + +
      • +
    • + +A-side: BSSAP/SCCP/M3UA (emulates MSC-side) + +
      • +
    • + +MGW side: MGCP (emulates MGW side) + +
      • +
    +
    • +
+
+
+osmodevcon2018-ttcn3_test_suites__2.png +
+
+
+
+
+

MSC_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +A: BSSAP/SCCP/M3UA (emulates BSC-side) + +
      • +
    • + +MNCC: MNCC/unix-domain (emulates ext. MNCC side) + +
      • +
    • + +MGW: MGCP (emulates MGW side) + +
      • +
    • + +GSUP (impllements HLR side) + +
      • +
    +
    • +
+
+
+osmodevcon2018-ttcn3_test_suites__3.png +
+
+
+
+
+

MGCP_Test.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +MGCP (emulates call agent) + +
      • +
    • + +RTP (stream source/sink) + +
      • +
    +
    • +
+
+
+
+

HLR_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +GSUP (emulates VLR/SGSN side) + +
      • +
    +
    • +
+
+
+
+

SIP_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +MNCC (emulates MSC side) + +
      • +
    • + +SIP (emulates SIP switch) + +
      • +
    +
    • +
+
+
+
+

SGSN_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +Gb (emulates PCU side NS/BSSGP + MS) + +
      • +
    • + +GSUP (emulates HLR) + +
      • +
    +
    • +
+
+
+
+

GGSN_Tests.ttcn

+
+
    +
  • + +external interfaces + +
      +
    • + +Gp: GTP (emulates SGSN) + +
      • +
    • + +Gi: IP (emulates Internet) + +
      • +
    +
    • +
+
+
+
+

Dockerized Setup

+
+
    +
  • + +one process per container + +
    • +
  • + +packages either + +
      +
    • + +IUT (e.g. osmo-bsc) + +
      • +
    • + +ATS (compiled docker test suite) + +
      • +
    • + +other utility (e.g. trxcon or osmo-bts-omldummy) + +
      • +
    +
    • +
  • + +why? + +
      +
    • + +no need for local ip/network configuration + +
      • +
    • + +standardized / packaged setup on every machine + +
      • +
    • + +run older/newer versions of ATS against older/newer IUT + +
      • +
    +
    • +
+
+
+
+

Jenkins CI Execution

+
+
    +
  1. + +update docker-playground.git + +
      +
    1. + +contains Dockerfile for ATS + IUT + +
      2. +
    +
    2. +
  2. + +rebuild IUT container[s] (e.g. osmo-bts-master) + +
      +
    1. + +git magic ensures re-build only if osmo-bts.git master changed + +
      2. +
    +
    3. +
  3. + +rebuild ATS container (e.g. ttcn3-bts-test) + +
      +
    1. + +git magic ensures re-build only if osmo-ttcn3-hacks.git master changed + +
      2. +
    +
    4. +
  4. + +run docker-playground/ttcn3-bts-test/jenkins.sh + +
      +
    1. + +creates docker network + +
      2. +
    2. + +starts IUT + ATS docker containers + +
      3. +
    3. + +collects test results + +
      4. +
    +
    5. +
+
+
+
+

Jenkins CI Reporting

+
+
    +
  • + +junit-xml generation + +
    • +
  • + +store artefacts + +
      +
    • + +pcap file of every test case + +
      • +
    • + +ATS log file (TTCN-3 testsuite) + +
      • +
    • + +IUT log file[s] (osmo-*.log) + +
      • +
    • + +IUT config file[s] (osmo-*.cfg) + +
      • +
    +
    • +
  • + +see https://jenkins.osmocom.org/jenkins/view/TTCN3/ + +
    • +
+
+
+
+

Further Reading

+
+ +
+
+
+

EOF

+
+

End of File

+
+
+ + diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__1.png b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__1.png new file mode 100644 index 0000000..45993c6 Binary files /dev/null and b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__1.png differ diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__2.png b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__2.png new file mode 100644 index 0000000..4c2a2cd Binary files /dev/null and b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__2.png differ diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__3.png b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__3.png new file mode 100644 index 0000000..ed1ae60 Binary files /dev/null and b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites__3.png differ -- cgit v1.2.3