From f1a0cedd3a0b170cbf04589348ef20741014e97c Mon Sep 17 00:00:00 2001
From: Harald Welte <laforge@gnumonks.org>
Date: Thu, 12 Apr 2018 15:33:48 +0200
Subject: osmodevcon2018 osmo-ttcn3 testsuites

---
 .../osmodevcon2018-ttcn3_test_suites.adoc          | 225 +++++++++++++++++++++
 1 file changed, 225 insertions(+)
 create mode 100644 2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc

(limited to '2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc')

diff --git a/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc
new file mode 100644
index 0000000..ae02b96
--- /dev/null
+++ b/2018/osmodevcon2018-ttcn3_test_suites/osmodevcon2018-ttcn3_test_suites.adoc
@@ -0,0 +1,225 @@
+Osmocom TTCN-3 Test Suites
+==========================
+:author:	Harald Welte <laforge@gnumonks.org>
+:copyright:	2018 by Harald Welte (License: CC-BY-SA)
+:backend:	slidy
+:max-width:	45em
+
+
+== Osmocom TTCN-3 Test Suites
+
+* developed in 2017+2018
+* compiled using Eclipse TITAN
+** uses just a command-line compiler + Makefiles
+** no IDE needed at all, don't let _Eclipse_ fool you
+* containerized in Docker
+* executed by Jenkins CI
+
+== Terminology
+
+ATS:: Abstract Test Suite
+MTC:: Main Test Component
+PTC:: Parallel Test Component
+IUT:: Implementation Under Test
+
+== Test Suite Philosophy
+
+* test one network element (our IUT)
+* test external behavior (3GPP and non-3GPP)
+* emulate entire environment from TTCN-3
+* don't reuse Osmocom C-code protocol implementations in the tests
+* test against independent TTCN-3 implementations!
+
+== What to test?
+
+* successful cases
+* erroneous cases (no answer, NACK, ...)
+** many difficult to reproduce with real phones/devices
+* load / resource exhaustion
+* spec compliance
+* focus on functionality actually relevant to IUT
+
+== Why TTCN-3 + TITAN
+
+* TTCN-3 specifically designed for telecom protocol testing
+* TITAN team released many telecom protocols in TTCN-3, such as
+** BSSAP, L3 (RR/MM/CC), SMS (CP/RP/TP), SS, M3UA, SCCP, GTP, NS, BSSGP, ...
+** shortens our test development cycle
+** permits us to test against known working industry implementations
+
+== Test suites for Osmocom CNI components
+
+* `osmo-bts`
+* `osmo-bsc`
+* `osmo-msc`
+* `osmo-mgw`
+* `osmo-hlr`
+* `osmo-sip-connector`
+* `osmo-sgsn`
+* `osmo-ggsn`
+
+== Test suites in progress
+
+* `osmo-pcu`
+* `osmo-bsc_nat`
+
+
+
+
+== BTS_Tests.ttcn
+
+* external interfaces
+** A-bis side: RSL (emulates BSC-side server)
+** Um side: L1CTL to control MS
+** PCU side: pcu_socket
+
+[graphviz]
+----
+digraph G {
+  rankdir=LR;
+  { rank=same; BTS, BSC};
+  BTS [label="IUT\nosmo-bts-trx",shape="box"];
+  ATS [label="ATS\nBTS_Tests.ttcn"];
+  BSC [label="osmo-bsc\nOML only"];
+  BTS -> fake_trx [label="bursts"];
+  fake_trx -> trxcon [label="bursts"];
+  trxcon -> ATS [label="bursts"];
+
+  BTS -> BSC [label="A-bis OML"];
+  BTS -> ATS [label="A-bis RSL"];
+
+  ATS -> BTS [label="pcu_sock"];
+  ATS -> BSC [label="VTY"];
+  ATS -> BTS [label="CTRL"];
+}
+----
+
+
+
+== BSC_Tests.ttcn
+
+* external interfaces
+** A-bis side: RSL (emulates BTS-side client)
+** A-side: BSSAP/SCCP/M3UA (emulates MSC-side)
+** MGW side: MGCP (emulates MGW side)
+
+[graphviz]
+----
+digraph G {
+  rankdir=LR;
+  { rank=same; BTS; STP; };
+  BSC [label="IUT\nosmo-bsc",shape="box"];
+  ATS [label="ATS\nBSC_Tests.ttcn"];
+  BTS [label="osmo-bts-omldummy\nOML only"];
+
+  BTS -> BSC [label="A-bis OML"];
+  ATS -> BSC [label="A-bis RSL"];
+  ATS -> BSC [label="CTRL"];
+  ATS -> BSC [label="VTY"];
+  ATS -> STP [label="A BSSAP\nSCCP/M3UA"];
+  BSC -> STP [label="A BSSAP\nSCCP/M3UA"];
+}
+----
+
+== MSC_Tests.ttcn
+
+* external interfaces
+** A: BSSAP/SCCP/M3UA (emulates BSC-side)
+** MNCC: MNCC/unix-domain (emulates ext. MNCC side)
+** MGW: MGCP (emulates MGW side)
+** GSUP (impllements HLR side)
+
+[graphviz]
+----
+digraph G {
+  rankdir=LR;
+  MSC [label="IUT\nosmo-msc",shape="box"];
+  ATS [label="ATS\nMSC_Tests.ttcn"];
+
+  ATS -> MSC [label="MNCC"];
+  ATS -> MSC [label="SMPP",style="dashed"];
+  ATS -> MSC [label="CTRL"];
+  ATS -> MSC [label="VTY"];
+  MSC -> ATS [label="GSUP"];
+  ATS -> STP [label="A BSSAP\nSCCP/M3UA"];
+  MSC -> STP [label="A BSSAP\nSCCP/M3UA"];
+}
+----
+
+
+== MGCP_Test.ttcn
+
+* external interfaces
+** MGCP (emulates call agent)
+** RTP (stream source/sink)
+
+== HLR_Tests.ttcn
+
+* external interfaces
+** GSUP (emulates VLR/SGSN side)
+
+== SIP_Tests.ttcn
+
+* external interfaces
+** MNCC (emulates MSC side)
+** SIP (emulates SIP switch)
+
+== SGSN_Tests.ttcn
+
+* external interfaces
+** Gb (emulates PCU side NS/BSSGP + MS)
+** GSUP (emulates HLR)
+
+== GGSN_Tests.ttcn
+
+* external interfaces
+** Gp: GTP (emulates SGSN)
+** Gi: IP (emulates Internet)
+
+
+== Dockerized Setup
+
+* one process per container
+* packages either
+** IUT (e.g. `osmo-bsc`)
+** ATS (compiled docker test suite)
+** other utility (e.g. `trxcon` or `osmo-bts-omldummy`)
+* why?
+** no need for local ip/network configuration
+** standardized / packaged setup on every machine
+** run older/newer versions of ATS against older/newer IUT
+
+== Jenkins CI Execution
+
+. update `docker-playground.git`
+.. contains `Dockerfile` for ATS + IUT
+. rebuild IUT container[s] (e.g. `osmo-bts-master`)
+.. git magic ensures re-build only if `osmo-bts.git` master changed
+. rebuild ATS container (e.g. `ttcn3-bts-test`)
+.. git magic ensures re-build only if `osmo-ttcn3-hacks.git` master changed
+. run `docker-playground/ttcn3-bts-test/jenkins.sh`
+.. creates docker network
+.. starts IUT + ATS docker containers
+.. collects test results
+
+
+== Jenkins CI Reporting
+
+* junit-xml generation
+* store artefacts
+** pcap file of every test case
+** ATS log file (TTCN-3 testsuite)
+** IUT log file[s] (`osmo-*.log`)
+** IUT config file[s] (`osmo-*.cfg`)
+* see https://jenkins.osmocom.org/jenkins/view/TTCN3/
+
+
+== Further Reading
+
+* http://git.osmocom.org/osmo-ttcn3-hacks/
+* http://git.osmocom.org/docker-playground/
+* http://osmocom.org/projects/cellular-infrastructure/wiki/Titan_TTCN3_Notes
+
+== EOF
+
+End of File
-- 
cgit v1.2.3

