Skip to content
This repository was archived by the owner on Aug 16, 2021. It is now read-only.

Commit 7c43db2

Browse files
dmiusdmius
committed
Collect /proc/meminfo data
1 parent 533f028 commit 7c43db2

File tree

1 file changed

+85
-72
lines changed

1 file changed

+85
-72
lines changed

‎nancy_run.sh‎

Lines changed: 85 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1325,6 +1325,65 @@ MACHINE_HOME="/machine_home/nancy_${CONTAINER_HASH}"
13251325
alias docker_exec='docker $DOCKER_CONFIG exec -i ${CONTAINER_HASH} '
13261326
get_system_characteristics
13271327

1328+
#######################################
1329+
# Stop postgres and wait for complete stop
1330+
# Globals:
1331+
# None
1332+
# Arguments:
1333+
# None
1334+
# Returns:
1335+
# None
1336+
#######################################
1337+
function stop_postgres {
1338+
dbg "Stopping Postgres..."
1339+
local cnt=0
1340+
while true; do
1341+
res=$(docker_exec bash -c "ps auxww | grep postgres | grep -v "grep" 2>/dev/null || echo ''")
1342+
if [[ -z "$res" ]]; then
1343+
# postgres process not found
1344+
dbg "Postgres stopped."
1345+
return;
1346+
fi
1347+
cnt=$((cnt+1))
1348+
if [[ "${cnt}" -ge "60" ]]; then
1349+
msg "WARNING: could not stop Postgres in 60 seconds. Killing."
1350+
docker_exec bash -c "sudo killall -s 9 postgres || true"
1351+
fi
1352+
# Try normal "fast stop"
1353+
docker_exec bash -c "sudo pg_ctlcluster ${PG_VERSION} main stop -m f || true"
1354+
sleep 1
1355+
done
1356+
}
1357+
1358+
#######################################
1359+
# Start postgres and wait for ready
1360+
# Globals:
1361+
# None
1362+
# Arguments:
1363+
# None
1364+
# Returns:
1365+
# None
1366+
#######################################
1367+
function start_postgres {
1368+
dbg "Starting Postgres..."
1369+
local cnt=0
1370+
while true; do
1371+
res=$(docker_exec bash -c "psql -Upostgres -d postgres -t -c \"select 1\" 2>/dev/null || echo '' ")
1372+
if [[ ! -z "$res" ]]; then
1373+
dbg "Postgres started."
1374+
return;
1375+
fi
1376+
cnt=$((cnt+1))
1377+
if [[ "${cnt}" -ge "60" ]]; then
1378+
dbg "WARNING: Can't start Postgres in 60 seconds." >&2
1379+
return 12
1380+
fi
1381+
docker_exec bash -c "sudo pg_ctlcluster ${PG_VERSION} main start || true"
1382+
sleep 1
1383+
done
1384+
dbg "Postgres started"
1385+
}
1386+
13281387
#######################################
13291388
# Extract the database backup from the attached EBS volume.
13301389
# Globals:
@@ -1424,8 +1483,7 @@ function rsync_backup(){
14241483
docker_exec bash -c "chown -R postgres:postgres /var/lib/postgresql" || true
14251484
docker_exec bash -c "chmod 0700 /var/lib/postgresql/$PG_VERSION/main/" || true
14261485

1427-
docker_exec bash -c "sudo /etc/init.d/postgresql start $VERBOSE_OUTPUT_REDIRECT"
1428-
sleep 10
1486+
start_postgres
14291487

14301488
if [[ ! -z ${DB_EBS_VOLUME_ID+x} ]] && [[ ! -z ${ORIGINAL_DB_NAME+x} ]] && [[ ! "$ORIGINAL_DB_NAME" == "test" ]]; then
14311489
docker_exec bash -c "psql --set ON_ERROR_STOP=on -U postgres -c 'drop database if exists test;'"
@@ -1464,8 +1522,8 @@ function attach_pgdata() {
14641522
local end_time=$(date +%s);
14651523
local duration=$(echo $((end_time-op_start_time)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
14661524
msg "Time taken to attach PGDATA: $duration."
1467-
docker_exec bash -c "sudo /etc/init.d/postgresql restart $VERBOSE_OUTPUT_REDIRECT"
1468-
sleep 30 # wait for Postgres to start, may take some time spent for recovery
1525+
stop_postgres
1526+
start_postgres
14691527
}
14701528

14711529
#######################################
@@ -1508,8 +1566,7 @@ if [[ "$RUN_ON" == "aws" ]]; then
15081566
dettach_db_ebs_drive
15091567
fi
15101568

1511-
docker_exec bash -c "sudo /etc/init.d/postgresql start ${VERBOSE_OUTPUT_REDIRECT}"
1512-
sleep 10 # wait for postgres started
1569+
start_postgres
15131570
else
15141571
if [[ ! -z ${DB_LOCAL_PGDATA+x} ]]; then
15151572
attach_pgdata true
@@ -1778,8 +1835,8 @@ function pg_config_init() {
17781835
local restart_needed=false
17791836
fi
17801837
if [[ $restart_needed == true ]]; then
1781-
docker_exec bash -c "sudo /etc/init.d/postgresql restart $VERBOSE_OUTPUT_REDIRECT"
1782-
sleep 10
1838+
stop_postgres
1839+
start_postgres
17831840
fi
17841841
END_TIME=$(date +%s)
17851842
DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
@@ -1804,9 +1861,8 @@ function apply_postgres_configuration() {
18041861
delta_config_filename=$(basename $delta_config)
18051862
docker_exec bash -c "echo '# DELTA:' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
18061863
docker_exec bash -c "cat $MACHINE_HOME/$delta_config_filename >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1807-
docker_exec bash -c "sudo /etc/init.d/postgresql restart $VERBOSE_OUTPUT_REDIRECT"
1808-
#msg $out
1809-
sleep 10
1864+
stop_postgres
1865+
start_postgres
18101866
END_TIME=$(date +%s);
18111867
DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
18121868
msg "Time taken to apply Postgres configuration delta: $DURATION."
@@ -2032,8 +2088,6 @@ function collect_results() {
20322088
docker_exec bash -c "gzip -c $LOG_PATH > $MACHINE_HOME/$ARTIFACTS_DIRNAME/postgresql.workload.$run_number.log.gz"
20332089
docker_exec bash -c "cp /etc/postgresql/$PG_VERSION/main/postgresql.conf $MACHINE_HOME/$ARTIFACTS_DIRNAME/postgresql.$run_number.conf"
20342090

2035-
save_artifacts
2036-
20372091
END_TIME=$(date +%s)
20382092
DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
20392093
msg "Time taken to generate and collect artifacts: $DURATION."
@@ -2156,6 +2210,21 @@ function start_monitoring {
21562210
>/dev/null 2>&1 </dev/null &"
21572211
ret_code="$?"
21582212
[[ "$ret_code" -ne "0" ]] && err "WARNING: Can't execute iostat"
2213+
2214+
# meminfo
2215+
memInfoMonitoring="#!/bin/bash\n\n \
2216+
while true; do\n \
2217+
date --rfc-3339=ns >> ${MACHINE_HOME}/${ARTIFACTS_DIRNAME}/meminfo.${run_number}.log\n \
2218+
cat /proc/meminfo >> ${MACHINE_HOME}/${ARTIFACTS_DIRNAME}/meminfo.${run_number}.log\n \
2219+
echo \"\" >> ${MACHINE_HOME}/${ARTIFACTS_DIRNAME}/meminfo.${run_number}.log\n \
2220+
sleep ${freq}\n \
2221+
done;
2222+
"
2223+
docker_exec bash -c "echo -e \"${memInfoMonitoring}\" > ${MACHINE_HOME}/meminfo.sh && chmod 700 ${MACHINE_HOME}/meminfo.sh"
2224+
docker_exec bash -c "nohup bash -c \"${MACHINE_HOME}/meminfo.sh\" >/dev/null 2>&1 </dev/null &"
2225+
ret_code="$?"
2226+
[[ "$ret_code" -ne "0" ]] && err "WARNING: Can't execute iostat"
2227+
21592228
set -e
21602229
}
21612230

@@ -2177,71 +2246,14 @@ function stop_monitoring {
21772246
docker_exec bash -c "killall mpstat >/dev/null 2>&1"
21782247
# iostat
21792248
docker_exec bash -c "killall iostat >/dev/null 2>&1"
2249+
# meminfo
2250+
docker_exec bash -c "killall meminfo.sh"
21802251
msg "Generating iostat graph..."
21812252
docker_exec bash -c "cd ${MACHINE_HOME}/${ARTIFACTS_DIRNAME} && iostat-cli --data iostat.${run_number}.log plot $VERBOSE_OUTPUT_REDIRECT || true"
21822253
docker_exec bash -c "mv ${MACHINE_HOME}/${ARTIFACTS_DIRNAME}/iostat.png ${MACHINE_HOME}/${ARTIFACTS_DIRNAME}/iostat.${run_number}.png $VERBOSE_OUTPUT_REDIRECT"
21832254
set -e
21842255
}
21852256

2186-
#######################################
2187-
# Stop postgres and wait for complete stop
2188-
# Globals:
2189-
# None
2190-
# Arguments:
2191-
# None
2192-
# Returns:
2193-
# None
2194-
#######################################
2195-
function stop_postgres {
2196-
dbg "Stopping Postgres..."
2197-
local cnt=0
2198-
while true; do
2199-
res=$(docker_exec bash -c "ps auxww | grep postgres | grep -v "grep" 2>/dev/null || echo ''")
2200-
if [[ -z "$res" ]]; then
2201-
# postgres process not found
2202-
dbg "Postgres stopped."
2203-
return;
2204-
fi
2205-
cnt=$((cnt+1))
2206-
if [[ "${cnt}" -ge "60" ]]; then
2207-
msg "WARNING: could not stop Postgres in 60 seconds. Killing."
2208-
docker_exec bash -c "sudo killall -s 9 postgres || true"
2209-
fi
2210-
# Try normal "fast stop"
2211-
docker_exec bash -c "sudo pg_ctlcluster ${PG_VERSION} main stop -m f "
2212-
sleep 1
2213-
done
2214-
}
2215-
2216-
#######################################
2217-
# Start postgres and wait for ready
2218-
# Globals:
2219-
# None
2220-
# Arguments:
2221-
# None
2222-
# Returns:
2223-
# None
2224-
#######################################
2225-
function start_postgres {
2226-
dbg "Starting Postgres..."
2227-
local cnt=0
2228-
while true; do
2229-
res=$(docker_exec bash -c "psql -Upostgres -d postgres -t -c \"select 1\" 2>/dev/null || echo '' ")
2230-
if [[ ! -z "$res" ]]; then
2231-
dbg "Postgres started."
2232-
return;
2233-
fi
2234-
cnt=$((cnt+1))
2235-
if [[ "${cnt}" -ge "60" ]]; then
2236-
dbg "WARNING: Can't start Postgres in 60 seconds." >&2
2237-
return 12
2238-
fi
2239-
docker_exec bash -c "sudo pg_ctlcluster ${PG_VERSION} main start || true"
2240-
sleep 1
2241-
done
2242-
dbg "Postgres started"
2243-
}
2244-
22452257
#######################################
22462258
# Do rollback to earlier created ZFS snapshot with stop and start postgres
22472259
# Globals:
@@ -2477,5 +2489,6 @@ while : ; do
24772489
[[ "$i" -eq "$runs_count" ]] && break;
24782490
done
24792491

2492+
save_artifacts
24802493
calc_estimated_cost
24812494
DONE=1

0 commit comments

Comments
 (0)