diff options
Diffstat (limited to '2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc')
| -rw-r--r-- | 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc | 225 |
1 files changed, 225 insertions, 0 deletions
diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc new file mode 100644 index 0000000..ae02b96 --- /dev/null +++ b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc @@ -0,0 +1,225 @@ +Osmocom TTCN-3 Test Suites +========================== +:author: Harald Welte <laforge@gnumonks.org> +:copyright: 2018 by Harald Welte (License: CC-BY-SA) +:backend: slidy +:max-width: 45em + + +== Osmocom TTCN-3 Test Suites + +* developed in 2017+2018 +* compiled using Eclipse TITAN +** uses just a command-line compiler + Makefiles +** no IDE needed at all, don't let _Eclipse_ fool you +* containerized in Docker +* executed by Jenkins CI + +== Terminology + +ATS:: Abstract Test Suite +MTC:: Main Test Component +PTC:: Parallel Test Component +IUT:: Implementation Under Test + +== Test Suite Philosophy + +* test one network element (our IUT) +* test external behavior (3GPP and non-3GPP) +* emulate entire environment from TTCN-3 +* don't reuse Osmocom C-code protocol implementations in the tests +* test against independent TTCN-3 implementations! + +== What to test? + +* successful cases +* erroneous cases (no answer, NACK, ...) +** many difficult to reproduce with real phones/devices +* load / resource exhaustion +* spec compliance +* focus on functionality actually relevant to IUT + +== Why TTCN-3 + TITAN + +* TTCN-3 specifically designed for telecom protocol testing +* TITAN team released many telecom protocols in TTCN-3, such as +** BSSAP, L3 (RR/MM/CC), SMS (CP/RP/TP), SS, M3UA, SCCP, GTP, NS, BSSGP, ... +** shortens our test development cycle +** permits us to test against known working industry implementations + +== Test suites for Osmocom CNI components + +* `osmo-bts` +* `osmo-bsc` +* `osmo-msc` +* `osmo-mgw` +* `osmo-hlr` +* `osmo-sip-connector` +* `osmo-sgsn` +* `osmo-ggsn` + +== Test suites in progress + +* `osmo-pcu` +* `osmo-bsc_nat` + + + + +== BTS_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BSC-side server) +** Um side: L1CTL to control MS +** PCU side: pcu_socket + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS, BSC}; + BTS [label="IUT\nosmo-bts-trx",shape="box"]; + ATS [label="ATS\nBTS_Tests.ttcn"]; + BSC [label="osmo-bsc\nOML only"]; + BTS -> fake_trx [label="bursts"]; + fake_trx -> trxcon [label="bursts"]; + trxcon -> ATS [label="bursts"]; + + BTS -> BSC [label="A-bis OML"]; + BTS -> ATS [label="A-bis RSL"]; + + ATS -> BTS [label="pcu_sock"]; + ATS -> BSC [label="VTY"]; + ATS -> BTS [label="CTRL"]; +} +---- + + + +== BSC_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BTS-side client) +** A-side: BSSAP/SCCP/M3UA (emulates MSC-side) +** MGW side: MGCP (emulates MGW side) + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS; STP; }; + BSC [label="IUT\nosmo-bsc",shape="box"]; + ATS [label="ATS\nBSC_Tests.ttcn"]; + BTS [label="osmo-bts-omldummy\nOML only"]; + + BTS -> BSC [label="A-bis OML"]; + ATS -> BSC [label="A-bis RSL"]; + ATS -> BSC [label="CTRL"]; + ATS -> BSC [label="VTY"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + BSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + +== MSC_Tests.ttcn + +* external interfaces +** A: BSSAP/SCCP/M3UA (emulates BSC-side) +** MNCC: MNCC/unix-domain (emulates ext. MNCC side) +** MGW: MGCP (emulates MGW side) +** GSUP (impllements HLR side) + +[graphviz] +---- +digraph G { + rankdir=LR; + MSC [label="IUT\nosmo-msc",shape="box"]; + ATS [label="ATS\nMSC_Tests.ttcn"]; + + ATS -> MSC [label="MNCC"]; + ATS -> MSC [label="SMPP",style="dashed"]; + ATS -> MSC [label="CTRL"]; + ATS -> MSC [label="VTY"]; + MSC -> ATS [label="GSUP"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + MSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + + +== MGCP_Test.ttcn + +* external interfaces +** MGCP (emulates call agent) +** RTP (stream source/sink) + +== HLR_Tests.ttcn + +* external interfaces +** GSUP (emulates VLR/SGSN side) + +== SIP_Tests.ttcn + +* external interfaces +** MNCC (emulates MSC side) +** SIP (emulates SIP switch) + +== SGSN_Tests.ttcn + +* external interfaces +** Gb (emulates PCU side NS/BSSGP + MS) +** GSUP (emulates HLR) + +== GGSN_Tests.ttcn + +* external interfaces +** Gp: GTP (emulates SGSN) +** Gi: IP (emulates Internet) + + +== Dockerized Setup + +* one process per container +* packages either +** IUT (e.g. `osmo-bsc`) +** ATS (compiled docker test suite) +** other utility (e.g. `trxcon` or `osmo-bts-omldummy`) +* why? +** no need for local ip/network configuration +** standardized / packaged setup on every machine +** run older/newer versions of ATS against older/newer IUT + +== Jenkins CI Execution + +. update `docker-playground.git` +.. contains `Dockerfile` for ATS + IUT +. rebuild IUT container[s] (e.g. `osmo-bts-master`) +.. git magic ensures re-build only if `osmo-bts.git` master changed +. rebuild ATS container (e.g. `ttcn3-bts-test`) +.. git magic ensures re-build only if `osmo-ttcn3-hacks.git` master changed +. run `docker-playground/ttcn3-bts-test/jenkins.sh` +.. creates docker network +.. starts IUT + ATS docker containers +.. collects test results + + +== Jenkins CI Reporting + +* junit-xml generation +* store artefacts +** pcap file of every test case +** ATS log file (TTCN-3 testsuite) +** IUT log file[s] (`osmo-*.log`) +** IUT config file[s] (`osmo-*.cfg`) +* see https://jenkins.osmocom.org/jenkins/view/TTCN3/ + + +== Further Reading + +* http://git.osmocom.org/osmo-ttcn3-hacks/ +* http://git.osmocom.org/docker-playground/ +* http://osmocom.org/projects/cellular-infrastructure/wiki/Titan_TTCN3_Notes + +== EOF + +End of File |