From c8a2480535c39d26062bd42f5dcf368382cd3670 Mon Sep 17 00:00:00 2001
From: MJB <mjb@it-innovation.soton.ac.uk>
Date: Mon, 22 Jan 2018 21:10:25 +0000
Subject: [PATCH] added template and updated simulator
---
.../influx/telegraf_ipendpoint_template.conf | 122 ++++++++++++++++++
src/mediaServiceSim/LineProtocolGenerator.py | 21 ++-
src/mediaServiceSim/simulator_v2.py | 121 +++++++++++------
3 files changed, 223 insertions(+), 41 deletions(-)
create mode 100644 scripts/influx/telegraf_ipendpoint_template.conf
diff --git a/scripts/influx/telegraf_ipendpoint_template.conf b/scripts/influx/telegraf_ipendpoint_template.conf
new file mode 100644
index 0000000..0fe2cc5
--- /dev/null
+++ b/scripts/influx/telegraf_ipendpoint_template.conf
@@ -0,0 +1,122 @@
+# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared inputs, and sent to the declared outputs.
+
+# Plugins must be declared in here to be active.
+# To deactivate a plugin, comment out the name and any variables.
+
+# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
+# file would generate.
+
+# Global tags can be specified here in key="value" format.
+[global_tags]
+ # location of the data centre
+ location={{LOCATION}}
+ # media service template id
+ sfc={{SFC_ID}}
+ # media service instance
+ sfc_i={{SFC_ID_INSTANCE}}
+ # service function type
+ sf={{SF_ID}}
+ # service function instance id
+ sf_i={{SF_ID_INSTANCE}}
+ # ipendpoint id aka surrogate instance
+ ipendpoint={{IP_ENDPOINT_ID}}
+
+# Configuration for telegraf agent
+[agent]
+ ## Default data collection interval for all inputs
+ interval = "10s"
+ ## Rounds collection interval to 'interval'
+ ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
+ round_interval = true
+
+ ## Telegraf will cache metric_buffer_limit metrics for each output, and will
+ ## flush this buffer on a successful write.
+ metric_buffer_limit = 1000
+ ## Flush the buffer whenever full, regardless of flush_interval.
+ flush_buffer_when_full = true
+
+ ## Collection jitter is used to jitter the collection by a random amount.
+ ## Each plugin will sleep for a random time within jitter before collecting.
+ ## This can be used to avoid many plugins querying things like sysfs at the
+ ## same time, which can have a measurable effect on the system.
+ collection_jitter = "0s"
+
+ ## Default flushing interval for all outputs. You shouldn't set this below
+ ## interval. Maximum flush_interval will be flush_interval + flush_jitter
+ flush_interval = "10s"
+ ## Jitter the flush interval by a random amount. This is primarily to avoid
+ ## large write spikes for users running a large number of telegraf instances.
+ ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+ flush_jitter = "0s"
+
+ ## Logging configuration:
+ ## Run telegraf in debug mode
+ debug = false
+ ## Run telegraf in quiet mode
+ quiet = false
+ ## Specify the log file name. The empty string means to log to stdout.
+ logfile = "G:/Telegraf/telegraf.log"
+
+ ## Override default hostname, if empty use os.Hostname()
+ hostname = ""
+
+
+###############################################################################
+# OUTPUTS #
+###############################################################################
+
+# Configuration for influxdb server to send metrics to
+[[outputs.influxdb]]
+ # The full HTTP or UDP endpoint URL for your InfluxDB instance.
+ # Multiple urls can be specified but it is assumed that they are part of the same
+ # cluster, this means that only ONE of the urls will be written to each interval.
+ # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
+ urls = ["{{INFLUXDB_URL}}"] # required
+ # The target database for metrics (telegraf will create it if not exists)
+ database = "CLMCMetrics" # required
+ # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+ # note: using second precision greatly helps InfluxDB compression
+ precision = "s"
+
+ ## Write timeout (for the InfluxDB client), formatted as a string.
+ ## If not provided, will default to 5s. 0s means no timeout (not recommended).
+ timeout = "5s"
+ # username = "telegraf"
+ # password = "metricsmetricsmetricsmetrics"
+ # Set the user agent for HTTP POSTs (can be useful for log differentiation)
+ # user_agent = "telegraf"
+ # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
+ # udp_payload = 512
+[[outputs.file]]
+ ## Files to write to, "stdout" is a specially handled file.
+ files = ["stdout", "/tmp/metrics.out"]
+
+ ## Data format to output.
+ ## Each data format has its own unique set of configuration options, read
+ ## more about them here:
+ ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+ data_format = "influx"
+
+
+###############################################################################
+# INPUTS #
+###############################################################################
+# # Influx HTTP write listener
+[[inputs.http_listener]]
+ ## Address and port to host HTTP listener on
+ service_address = ":8186"
+
+ ## timeouts
+ read_timeout = "10s"
+ write_timeout = "10s"
+
+ ## HTTPS
+ #tls_cert= "/etc/telegraf/cert.pem"
+ #tls_key = "/etc/telegraf/key.pem"
+
+ ## MTLS
+ #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
+
\ No newline at end of file
diff --git a/src/mediaServiceSim/LineProtocolGenerator.py b/src/mediaServiceSim/LineProtocolGenerator.py
index a8ccb92..3d4b077 100644
--- a/src/mediaServiceSim/LineProtocolGenerator.py
+++ b/src/mediaServiceSim/LineProtocolGenerator.py
@@ -18,7 +18,7 @@ def generate_network_report(recieved_bytes, sent_bytes, time):
result += ' ' + str(_getNSTime(time))
# Measurement
- print(result)
+ #print(result)
return result
@@ -43,7 +43,7 @@ def generate_vm_config(state, cpu, mem, storage, time):
# Reports cpu usage, scaling on requests
def generate_cpu_report(cpu_usage, cpu_active_time, cpu_idle_time, time):
- result = 'vm_host_cpu_usage'
+ result = 'cpu_usage'
# Tag
result += ' '
# field
@@ -74,6 +74,21 @@ def generate_mpegdash_report(resource, requests, avg_response_time, peak_respons
print(result)
return result
+#ipendpoint_route,ipendpoint_id,cont_nav=FQDN HTTP_REQUESTS_FQDN_M, NETWORK_FQDN_LATENCY timestamp
+def generate_ipendpoint_route(resource, requests, latency, time):
+ # Measurement
+ result = 'ipendpoint_route'
+ # Tags
+ result += ',cont_nav=\"' + str(resource) + "\" "
+ # Fields
+
+ # result += 'cont_rep=' + str(quality) + ','
+ result += 'http_requests_fqdn_m=' + str(requests) + ','
+ result += 'network_fqdn_latency=' + str(latency)
+ # Timestamp
+ result += ' ' + str(_getNSTime(time))
+ #print(result)
+ return result
# Influx needs strings to be quoted, this provides a utility interface to do this
def quote_wrap(str):
@@ -84,7 +99,7 @@ def quote_wrap(str):
def _getNSTime(time):
# Convert to nano-seconds
timestamp = int(1000000000*time)
- print("timestamp", timestamp)
+ #print("timestamp", timestamp)
return timestamp
# DEPRICATED
diff --git a/src/mediaServiceSim/simulator_v2.py b/src/mediaServiceSim/simulator_v2.py
index 8cf7a9e..ac8b0c5 100644
--- a/src/mediaServiceSim/simulator_v2.py
+++ b/src/mediaServiceSim/simulator_v2.py
@@ -7,11 +7,14 @@ import random
# Simulation parameters
TICK_TIME = 1
-DEFAULT_REQUEST_RATE_INCREMENT = 5
-SIMULATION_TIME_SEC = 180
+DEFAULT_REQUEST_RATE_INC = 1
+DEFAULT_REQUEST_RATE_INC_PERIOD = 10
+SIMULATION_TIME_SEC = 60*60
# CLMC parameters
INFLUX_DB_URL = 'http://192.168.50.10:8086'
+AGENT_URL1 = 'http://192.168.50.11:8186'
+AGENT_URL2 = 'http://192.168.50.12:8186'
# Simulator for services
class sim:
@@ -28,54 +31,47 @@ class sim:
start_time = time.time()-SIMULATION_TIME_SEC
sim_time = start_time
- ip_endpoints = [{'agent_url': 'http://192.168.50.11:8186', 'location': 'DC1', 'cpu': 16,
- 'mem': '8GB', 'storage': '1TB', 'request_queue': 0, 'request_arrival_rate': 0},
- {'agent_url': 'http://192.168.50.12:8186', 'location': 'DC2', 'cpu': 2,
- 'mem': '8GB', 'storage': '1TB', 'request_queue': 0, 'request_arrival_rate': 0}
+ # segment_size : the length of video requested at a time
+ # bit_rate: MPEG-2 High 1080p 25fps = 80Mbps
+ ip_endpoints = [{'agent_url': AGENT_URL1, 'location': 'DC1', 'cpu': 16,
+ 'mem': '8GB', 'storage': '1TB', 'request_queue': 0, 'request_arrival_rate': 0,
+ 'segment_size': 2, 'video_bit_rate': 80, 'packet_size': 1500},
+ {'agent_url': AGENT_URL2, 'location': 'DC2', 'cpu': 4,
+ 'mem': '8GB', 'storage': '1TB', 'request_queue': 0, 'request_arrival_rate': 0,
+ 'segment_size': 2, 'video_bit_rate': 80, 'packet_size': 1500}
]
# Simulate configuration of the ipendpoints
# endpoint state->mu, sigma, secs normal distribution
- config_delay_dist = {"placed": [5, 0.68], "booted": [10, 0.68],"connected": [10, 0.68]}
+ config_delay_dist = {"placing": [10, 0.68], "booting": [10, 0.68],"connecting": [10, 0.68]}
- # Place the endpoints
- max_delay = 0
+ # Place endpoints
+ max_delay = 0
for ip_endpoint in ip_endpoints:
- delay_time = random.normalvariate(config_delay_dist['placed'][0], config_delay_dist['placed'][0]*config_delay_dist['placed'][1])
-
- # print('sim_time: ', sim_time)
- # print('delay_time: ', delay_time)
- # timestamp = sim_time+delay_time
- # print('timestamp: ', timestamp)
- # ns_time = int(timestamp*1000000)
- # print('ns_time: ', ns_time)
-
- self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_vm_config('placed', ip_endpoint['cpu'], ip_endpoint['mem'], ip_endpoint['storage'], sim_time+delay_time))
+ delay_time = self._changeVMState(sim_time, ip_endpoint, config_delay_dist['placing'][0], config_delay_dist['placing'][0]*config_delay_dist['placing'][1], 'placing', 'placed')
if delay_time > max_delay:
max_delay = delay_time
-
sim_time +=max_delay
+
+ # Boot endpoints
max_delay = 0
- # Boot the endpoints
for ip_endpoint in ip_endpoints:
- delay_time = random.normalvariate(config_delay_dist['booted'][0], config_delay_dist['booted'][0]*config_delay_dist['booted'][1])
- self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_vm_config('booted', ip_endpoint['cpu'], ip_endpoint['mem'], ip_endpoint['storage'], sim_time+delay_time))
+ delay_time = self._changeVMState(sim_time, ip_endpoint, config_delay_dist['booting'][0], config_delay_dist['booting'][0]*config_delay_dist['booting'][1], 'booting', 'booted')
if delay_time > max_delay:
max_delay = delay_time
-
sim_time +=max_delay
+
+ # Connect endpoints
max_delay = 0
- # Connect the endpoints
for ip_endpoint in ip_endpoints:
- delay_time = random.normalvariate(config_delay_dist['connected'][0], config_delay_dist['connected'][0]*config_delay_dist['connected'][1])
- self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_vm_config('connected', ip_endpoint['cpu'], ip_endpoint['mem'], ip_endpoint['storage'], sim_time+delay_time))
+ delay_time = self._changeVMState(sim_time, ip_endpoint, config_delay_dist['connecting'][0], config_delay_dist['connecting'][0]*config_delay_dist['connecting'][1], 'connecting', 'connected')
if delay_time > max_delay:
max_delay = delay_time
-
sim_time +=max_delay
- request_arrival_rate_inc = DEFAULT_REQUEST_RATE_INCREMENT
+ request_arrival_rate_inc = DEFAULT_REQUEST_RATE_INC
request_queue = 0
+ inc_period_count = 0
for i in range(simulation_length_seconds):
for ip_endpoint in ip_endpoints:
request_processing_time = 0
@@ -90,17 +86,24 @@ class sim:
peak_response_time = 0
# linear inc to arrival rate
- ip_endpoint['request_arrival_rate'] += request_arrival_rate_inc
+ if inc_period_count >= DEFAULT_REQUEST_RATE_INC_PERIOD:
+ ip_endpoint['request_arrival_rate'] += request_arrival_rate_inc
+ inc_period_count = 0
+ else:
+ inc_period_count += 1
# add new requests to the queue
ip_endpoint['request_queue'] += ip_endpoint['request_arrival_rate']
- # time to process one request (mS) in the current second
- request_processing_time = int(random.normalvariate(50, 50*0.95))
+ # time to process one second of video (mS) in the current second
+ request_processing_time = int(random.normalvariate(10, 10*0.68))
if request_processing_time <= 10:
request_processing_time = 10
+ # time depends on the length of the segments in seconds
+ request_processing_time *= ip_endpoint['segment_size']
+
# amount of cpu time (mS) per tick
cpu_time_available = ip_endpoint['cpu']*TICK_TIME*1000
- max_requests_processed = cpu_time_available/request_processing_time
+ max_requests_processed = int(cpu_time_available/request_processing_time)
# calc how many requests processed
if ip_endpoint['request_queue'] <= max_requests_processed:
# processed all of the requests
@@ -108,16 +111,17 @@ class sim:
else:
# processed the maxmum number of requests
requests_processed = max_requests_processed
+
# calculate cpu usage
cpu_active_time = int(requests_processed*request_processing_time)
cpu_idle_time = int(cpu_time_available-cpu_active_time)
cpu_usage = cpu_active_time/cpu_time_available
self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_cpu_report(cpu_usage, cpu_active_time, cpu_idle_time, sim_time))
- # calc network usage metrics with no constraints.
- bytes_sent = 1024*requests_processed
- bytes_rec = 32*requests_processed
- self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_network_report(bytes_rec, bytes_sent, sim_time))
+ # calc network usage metrics
+ bytes_rx = 2048*requests_processed
+ bytes_tx = int(ip_endpoint['video_bit_rate']/8*1000000*requests_processed*ip_endpoint['segment_size'])
+ self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_network_report(bytes_rx, bytes_tx, sim_time))
# time to process all of the requests in the queue
peak_response_time = ip_endpoint['request_queue']*request_processing_time/ip_endpoint['cpu']
@@ -125,6 +129,16 @@ class sim:
avg_response_time = (peak_response_time+request_processing_time)/2
self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_mpegdash_report('https://Netflix.com/scream', ip_endpoint['request_arrival_rate'], avg_response_time, peak_response_time, sim_time))
+ # need to calculate this but sent at 5mS for now
+ network_request_delay = 0.005
+
+ # calculate network response delays (2km link, 100Mbps)
+ network_response_delay = self._calcNetworkDelay(2000, 100, ip_endpoint['packet_size'], ip_endpoint['video_bit_rate'], ip_endpoint['segment_size'])
+
+ e2e_delay = network_request_delay + (avg_response_time/1000) + network_response_delay
+
+ self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_ipendpoint_route('https://Netflix.com/scream', ip_endpoint['request_arrival_rate'], e2e_delay, sim_time))
+
# remove requests processed off the queue
ip_endpoint['request_queue'] -= int(requests_processed)
@@ -132,6 +146,38 @@ class sim:
end_time = sim_time
print("Simulation Finished. Start time {0}. End time {1}. Total time {2}".format(start_time,end_time,end_time-start_time))
+ # distance metres
+ # bandwidth Mbps
+ # package size bytes
+ # tx_video_bit_rate bp/sec
+ # segment size sec
+ def _calcNetworkDelay(self, distance, bandwidth, packet_size, tx_video_bit_rate, segment_size):
+ response_delay = 0
+
+ # propogation delay = distance/speed () (e.g 2000 metres * 2*10^8 for optical fibre)
+ propogation_delay = distance/(2*100000000)
+ # packetisation delay = ip packet size (bits)/tx rate (e.g. 100Mbp with 0% packet loss)
+ packetisation_delay = (packet_size*8)/(bandwidth*1000000)
+ # print('packetisation_delay:', packetisation_delay)
+ # total number of packets to be sent
+ packets = (tx_video_bit_rate*1000000)/(packet_size*8)
+ # print('packets:', packets)
+ response_delay = packets*(propogation_delay+packetisation_delay)
+ # print('response_delay:', response_delay)
+
+ return response_delay
+
+ def _changeVMState(self, sim_time, ip_endpoint, mu, sigma, transition_state, next_state):
+ delay_time = 0
+
+ self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_vm_config(transition_state, ip_endpoint['cpu'], ip_endpoint['mem'], ip_endpoint['storage'], sim_time))
+
+ delay_time = random.normalvariate(mu, sigma)
+
+ self._sendInfluxData(ip_endpoint['agent_url'], lp.generate_vm_config(next_state, ip_endpoint['cpu'], ip_endpoint['mem'], ip_endpoint['storage'], sim_time+delay_time))
+
+ return delay_time
+
def _createDB(self):
self._sendInfluxQuery(self.influx_url, 'CREATE DATABASE ' + self.influx_db)
@@ -152,7 +198,6 @@ class sim:
req = urllib.request.Request(url + '/write?db=' + self.influx_db, data, header)
urllib.request.urlopen(req)
-
simulator = sim(INFLUX_DB_URL)
simulator.run(SIMULATION_TIME_SEC)
--
GitLab