From a8dc590f567c8e6f846b8c2e80827475ea3bae29 Mon Sep 17 00:00:00 2001 From: Harald Welte Date: Thu, 18 Oct 2018 00:55:30 +0200 Subject: add osmocon2018-testing_initiative --- .../osmocon2018-testing_initiative.adoc | 360 +++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 2018/osmocon2018-testing_initiative/osmocon2018-testing_initiative.adoc diff --git a/2018/osmocon2018-testing_initiative/osmocon2018-testing_initiative.adoc b/2018/osmocon2018-testing_initiative/osmocon2018-testing_initiative.adoc new file mode 100644 index 0000000..fcd91e7 --- /dev/null +++ b/2018/osmocon2018-testing_initiative/osmocon2018-testing_initiative.adoc @@ -0,0 +1,360 @@ +Osmocom Testing Initiative +========================== +:author: Harald Welte +:copyright: 2018 by Harald Welte (License: CC-BY-SA) +:backend: slidy +:max-width: 45em + +== Osmocom testing until mid-2017 + +* manual testing +* unit tests for some small portion of mostly utility functions +** `make check` in the respective repository exceutes them +* VTY interface interactive testing using specific python test code + +== split NITB aftermath (the good parts) + +* biggest architectural change since we started in 2008 +* lots of good reasons and design improvements +** finite state machines with proper timeouts / clean-up +** proper 3GPP AoIP with interoperability tesing +** no synchronous HLR database access +** HLR access from OsmoMSC and OsmoSGSN +** 2G/3G authentication over GERAN and UTRAN + +== split NITB aftermath (the bad parts) + +[role="incremental"] +* never-ending list of breakage +** actual regressions of things that used to work before +** things that were _known omissions_ during the restructuring +* some commercial users stuck with SCCPlite and thus old @osmo-bsc-sccplite@ +** almost none of the new features or bug fixes there +** no automatic testing +** back-ports time-consuming + +== split NITB aftermath (lessons learned) + +* overall complexity of Osomcoom cellular is quite stunning now +* absence of proper functional testing has caused massive fall-out +* the split architecture allows for betteer testing of smaller parts of the system +* my personal main focus of the last 11+ months: +[role="incremental"] +** testing, testing, testing, testing +** testing, testing, testing, testing +** some more testing +** even more testing + + +== Osmocom CNI testing (1/2) + +[role="incremental"] +* unit test (autotest, like we always had) +** test individual functions / APIs of libraries / programs +** executed during "make check" and hence before any patch can get merged +* automatized functional tests in TTCN-3 +** test _external_ visible behavior on interfaces such as Abis, A, GSUP, GTP, MNCC, PCUIF, CTRL, VTY, ... +** executed nightly by Jenkins (could be more frequently) + +== Osmocom CNI testing (2/2) + +[role="incremental"] +* osmo-gsm-tester +** tests entire Osmoocom network with BTS/BSC/MSC/HLR/PCU/SGSN/GGSN/... +** uses real BTS + MS hardware (over coaxial cable) +** automatic execution multiple times per day +* interop tests +** against NG40 RAN + CN simulator from NG4% (A / Gb / Iu level) +** not fully automatized yet + + + +== Osmocom TTCN-3 Test Suites + +* developed in 2017+2018 +* compiled using Eclipse TITAN +** uses just a command-line compiler + Makefiles +** no IDE needed at all, don't let _Eclipse_ fool you +* containerized in Docker +* executed by Jenkins CI + +== Terminology + +ATS:: Abstract Test Suite +MTC:: Main Test Component +PTC:: Parallel Test Component +IUT:: Implementation Under Test + +== Test Suite Philosophy + +* test one network element (our IUT) +* test external behavior (3GPP and non-3GPP) +* emulate entire environment from TTCN-3 +* don't reuse Osmocom C-code protocol implementations in the tests +* test against independent TTCN-3 implementations! + +== What to test? + +* successful cases +* erroneous cases (no answer, NACK, ...) +** many difficult to reproduce with real phones/devices +* load / resource exhaustion +* spec compliance +* focus on functionality actually relevant to IUT + +== Why TTCN-3 + TITAN + +* TTCN-3 specifically designed for telecom protocol testing +* TITAN team released many telecom protocols in TTCN-3, such as +** BSSAP, L3 (RR/MM/CC), SMS (CP/RP/TP), SS, M3UA, SCCP, GTP, NS, BSSGP, ... +** shortens our test development cycle +** permits us to test against known working industry implementations + +== Test suites for Osmocom CNI components + +* `osmo-bts` +* `osmo-bsc` (for both AoIP and SCCPlite) +* `osmo-msc` +* `osmo-mgw` +* `osmo-hlr` +* `osmo-sip-connector` +* `osmo-sgsn` +* `osmo-ggsn` + +== Test suites in progress + +* `osmo-pcu` +* `osmo-bsc_nat` +* `osmo-gbproxy` + + + + +== BTS_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BSC-side server) +** Um side: L1CTL to control MS +** PCU side: pcu_socket + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS, BSC}; + BTS [label="IUT\nosmo-bts-trx",shape="box"]; + ATS [label="ATS\nBTS_Tests.ttcn"]; + BSC [label="osmo-bsc\nOML only"]; + BTS -> fake_trx [label="bursts"]; + fake_trx -> trxcon [label="bursts"]; + trxcon -> ATS [label="bursts"]; + + BTS -> BSC [label="A-bis OML"]; + BTS -> ATS [label="A-bis RSL"]; + + ATS -> BTS [label="pcu_sock"]; + ATS -> BSC [label="VTY"]; + ATS -> BTS [label="CTRL"]; +} +---- + + + +== BSC_Tests.ttcn + +* external interfaces +** A-bis side: RSL (emulates BTS-side client) +** A-side: BSSAP/SCCP/M3UA (emulates MSC-side) +** MGW side: MGCP (emulates MGW side) + +[graphviz] +---- +digraph G { + rankdir=LR; + { rank=same; BTS; STP; }; + BSC [label="IUT\nosmo-bsc",shape="box"]; + ATS [label="ATS\nBSC_Tests.ttcn"]; + BTS [label="osmo-bts-omldummy\nOML only"]; + + BTS -> BSC [label="A-bis OML"]; + ATS -> BSC [label="A-bis RSL"]; + ATS -> BSC [label="CTRL"]; + ATS -> BSC [label="VTY"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + BSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + +== MSC_Tests.ttcn + +* external interfaces +** A: BSSAP/SCCP/M3UA (emulates BSC-side) +** MNCC: MNCC/unix-domain (emulates ext. MNCC side) +** MGW: MGCP (emulates MGW side) +** GSUP (impllements HLR side) + +[graphviz] +---- +digraph G { + rankdir=LR; + MSC [label="IUT\nosmo-msc",shape="box"]; + ATS [label="ATS\nMSC_Tests.ttcn"]; + + ATS -> MSC [label="MNCC"]; + ATS -> MSC [label="SMPP",style="dashed"]; + ATS -> MSC [label="CTRL"]; + ATS -> MSC [label="VTY"]; + MSC -> ATS [label="GSUP"]; + ATS -> STP [label="A BSSAP\nSCCP/M3UA"]; + MSC -> STP [label="A BSSAP\nSCCP/M3UA"]; +} +---- + + +== MGCP_Test.ttcn + +* external interfaces +** MGCP (emulates call agent) +** RTP (stream source/sink) + +[graphviz] +---- +digraph G { + rankdir=LR; + MGW [label="IUT\nosmo-mgw",shape="box"]; + ATS [label="ATS\nMGCP_Test.ttcn"]; + + ATS -> MGW [label="RTP"]; + ATS -> MGW [label="MGCP"]; + MGW -> ATS [label="RTP"]; +} +---- + +== HLR_Tests.ttcn + +* external interfaces +** GSUP (emulates VLR/SGSN side) +** VTY + +[graphviz] +---- +digraph G { + rankdir=LR; + HLR [label="IUT\nosmo-hlr",shape="box"]; + ATS [label="ATS\nHLR_Tests.ttcn"]; + + ATS -> HLR [label="GSUP"]; + ATS -> HLR [label="VTY"]; +} +---- + + +== SIP_Tests.ttcn + +* external interfaces +** MNCC (emulates MSC side) +** SIP (emulates SIP switch) +** VTY + +[graphviz] +---- +digraph G { + rankdir=LR; + SIP [label="SIP\nosmo-sip-connector",shape="box"]; + ATS [label="ATS\nSIP_Tests.ttcn"]; + + ATS -> SIP [label="MNCC"]; + ATS -> SIP [label="SIP"]; + ATS -> SIP [label="VTY"]; +} +---- + + +== SGSN_Tests.ttcn + +* external interfaces +** Gb (emulates PCU side NS/BSSGP + MS) +** GSUP (emulates HLR) +** VTY + +[graphviz] +---- +digraph G { + rankdir=LR; + SGSN [label="SGSN\nosmo-sgsn",shape="box"]; + ATS [label="ATS\nSGSN_Tests.ttcn"]; + + ATS -> SGSN [label="Gb"]; + SGSN-> ATS [label="Gp (GTP)"]; + ATS -> SGSN [label="VTY"]; +} +---- + + +== GGSN_Tests.ttcn + +* external interfaces +** Gp: GTP (emulates SGSN) +** Gi: IP (emulates Internet) + +[graphviz] +---- +digraph G { + rankdir=LR; + GGSN [label="GGSN\nosmo-ggsn",shape="box"]; + ATS [label="ATS\nGGSN_Tests.ttcn"]; + + ATS -> GGSN [label="Gp (GTP)"]; + GGSN -> ATS [label="Gi (IP)"]; + ATS -> GGSN [label="VTY"]; +} +---- + + + +== Dockerized Setup + +* one process per container +* packages either +** IUT (e.g. `osmo-bsc`) +** ATS (compiled docker test suite) +** other utility (e.g. `trxcon` or `osmo-bts-omldummy`) +* why? +** no need for local ip/network configuration +** standardized / packaged setup on every machine +** run older/newer versions of ATS against older/newer IUT + +== Jenkins CI Execution + +. update `docker-playground.git` +.. contains `Dockerfile` for ATS + IUT +. rebuild IUT container[s] (e.g. `osmo-bts-master`) +.. git magic ensures re-build only if `osmo-bts.git` master changed +. rebuild ATS container (e.g. `ttcn3-bts-test`) +.. git magic ensures re-build only if `osmo-ttcn3-hacks.git` master changed +. run `docker-playground/ttcn3-bts-test/jenkins.sh` +.. creates docker network +.. starts IUT + ATS docker containers +.. collects test results + + +== Jenkins CI Reporting + +* junit-xml generation +* store artefacts +** pcap file of every test case +** ATS log file (TTCN-3 testsuite) +** IUT log file[s] (`osmo-*.log`) +** IUT config file[s] (`osmo-*.cfg`) +* see https://jenkins.osmocom.org/jenkins/view/TTCN3/ + + +== Further Reading + +* http://git.osmocom.org/osmo-ttcn3-hacks/ +* http://git.osmocom.org/docker-playground/ +* http://osmocom.org/projects/cellular-infrastructure/wiki/Titan_TTCN3_Notes + +== EOF + +End of File -- cgit v1.2.3