625 lines
21 KiB
HTML
625 lines
21 KiB
HTML
<!DOCTYPE html>
|
|
<!--[if lt IE 7 ]><html class="no-js ie6"><![endif]-->
|
|
<!--[if IE 7 ]><html class="no-js ie7"><![endif]-->
|
|
<!--[if IE 8 ]><html class="no-js ie8"><![endif]-->
|
|
<!--[if IE 9 ]><html class="no-js ie9"><![endif]-->
|
|
<!--[if (gt IE 9)|!(IE)]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<title>Scribe - Spearhead Systems Incident Response Documentation</title>
|
|
|
|
<!-- Author and License -->
|
|
<meta name="author" content="Spearhead Systems, Inc." />
|
|
<meta name="dcterms.license" content="http://www.apache.org/licenses/LICENSE-2.0" />
|
|
|
|
<!-- Page Description -->
|
|
|
|
<meta name="keywords" content="spearhead, incident, response" />
|
|
<meta name="robots" content="index, follow, noarchive" />
|
|
|
|
<!-- Mobile -->
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0" />
|
|
<meta name="theme-color" content="#1f293a" />
|
|
|
|
<!-- Canonical Link -->
|
|
<link rel="canonical" href="https://response.spearhead.systems/training/scribe/">
|
|
|
|
<!-- Favicon -->
|
|
|
|
<link rel="shortcut icon" type="image/x-icon" href="../../assets/img/icon.png" />
|
|
<link rel="icon" type="image/x-icon" href="../../assets/img/icon.png" />
|
|
|
|
<!-- Apple -->
|
|
<meta name="apple-mobile-web-app-title" content="Spearhead Systems Incident Response Documentation" />
|
|
<meta name="apple-mobile-web-app-capable" content="yes" />
|
|
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
|
<link rel="apple-touch-icon" href="../../assets/img/icon.png">
|
|
|
|
<!-- Open Graph -->
|
|
<meta property="og:url" content="https://response.spearhead.systems/training/scribe/" />
|
|
<meta property="og:title" content="Scribe - Spearhead Systems Incident Response Documentation" />
|
|
<meta property="og:site_name" content="Spearhead Systems Incident Response Documentation" />
|
|
<meta property="og:description" content="A collection of information about the Spearhead Systems incident response process. Not only how to prepare new employees for on-call responsibilities, but also how to handle major incidents, both in preparation and after-work." />
|
|
<meta property="og:image" content="https://response.spearhead.systems/assets/img/cover.png" />
|
|
<meta property="og:locale" content="en_US" />
|
|
<meta property="og:type" content="website" />
|
|
|
|
<!-- Twitter -->
|
|
<meta name="twitter:card" content="summary_large_image" />
|
|
<meta name="twitter:title" content="Scribe - Spearhead Systems Incident Response Documentation" />
|
|
<meta name="twitter:description" content="A collection of information about the Spearhead Systems incident response process. Not only how to prepare new employees for on-call responsibilities, but also how to handle major incidents, both in preparation and after-work." />
|
|
<meta name="twitter:image" content="https://response.spearhead.systems/assets/img/cover.png" />
|
|
|
|
<!-- Style -->
|
|
<style>
|
|
@font-face {
|
|
font-family: 'Icon';
|
|
src: url('../../assets/fonts/icon.eot?52m981');
|
|
src: url('../../assets/fonts/icon.eot?#iefix52m981')
|
|
format('embedded-opentype'),
|
|
url('../../assets/fonts/icon.woff?52m981')
|
|
format('woff'),
|
|
url('../../assets/fonts/icon.ttf?52m981')
|
|
format('truetype'),
|
|
url('../../assets/fonts/icon.svg?52m981#icon')
|
|
format('svg');
|
|
font-weight: normal;
|
|
font-style: normal;
|
|
}
|
|
</style>
|
|
<link rel="stylesheet" href="../../assets/stylesheets/application-a422ff04cc.css">
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/palettes-05ab2406df.css">
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Colfax Regular:400,700|Roboto+Mono">
|
|
<style>
|
|
body, input {
|
|
font-family: 'Colfax Regular', Helvetica, Arial, sans-serif;
|
|
}
|
|
pre, code {
|
|
font-family: 'Roboto Mono', 'Courier New', 'Courier', monospace;
|
|
}
|
|
</style>
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/css/extra.css">
|
|
|
|
|
|
<!-- Scripts -->
|
|
<script src="../../assets/javascripts/modernizr-4ab42b99fd.js"></script>
|
|
|
|
</head>
|
|
|
|
|
|
|
|
<body class="palette-primary-green palette-accent-blue-grey">
|
|
|
|
|
|
|
|
|
|
<div class="backdrop">
|
|
<div class="backdrop-paper"></div>
|
|
</div>
|
|
<input class="toggle" type="checkbox" id="toggle-drawer">
|
|
<input class="toggle" type="checkbox" id="toggle-search">
|
|
<label class="toggle-button overlay" for="toggle-drawer"></label>
|
|
<header class="header">
|
|
<nav aria-label="Header">
|
|
<div class="bar default">
|
|
<div class="button button-menu" role="button" aria-label="Menu">
|
|
<label class="toggle-button icon icon-menu" for="toggle-drawer">
|
|
<span></span>
|
|
</label>
|
|
</div>
|
|
<div class="stretch">
|
|
<div class="mainlogo">
|
|
<a href="/" title="Go to homepage.">
|
|
<img src="../../assets/img/logo.png" title="PagerDuty" />
|
|
</a>
|
|
</div>
|
|
<div class="title">
|
|
<span class="path">
|
|
Incident Response
|
|
<i class="icon icon-link"></i>
|
|
</span>
|
|
|
|
<span class="path">
|
|
|
|
|
|
Training <i class="icon icon-link"></i>
|
|
|
|
|
|
</span>
|
|
|
|
Scribe
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="button button-twitter" role="button" aria-label="Twitter">
|
|
<a href="https://twitter.com/spearhead_sys" title="@spearhead_sys on Twitter" target="_blank" class="toggle-button icon icon-twitter"></a>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="button button-github" role="button" aria-label="GitHub">
|
|
<a href="https://github.com/spearheadsys" title="@spearheadsys on GitHub" target="_blank" class="toggle-button icon icon-github"></a>
|
|
</div>
|
|
|
|
<div class="button button-search" role="button" aria-label="Search">
|
|
<label class="toggle-button icon icon-search" title="Search" for="toggle-search"></label>
|
|
</div>
|
|
</div>
|
|
<div class="bar search">
|
|
<div class="button button-close" role="button" aria-label="Close">
|
|
<label class="toggle-button icon icon-back" for="toggle-search"></label>
|
|
</div>
|
|
<div class="stretch">
|
|
<div class="field">
|
|
<input class="query" type="text" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false">
|
|
</div>
|
|
</div>
|
|
<div class="button button-reset" role="button" aria-label="Search">
|
|
<button class="toggle-button icon icon-close" id="reset-search"></button>
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
</header>
|
|
<main class="main">
|
|
|
|
<div class="drawer">
|
|
<nav aria-label="Navigation">
|
|
|
|
<a href="https://github.com/spearheadsys/issue-response-docs" class="project">
|
|
<!-- <div class="banner">
|
|
|
|
<div class="logo">
|
|
<img src="../../assets/img/icon.png">
|
|
</div>
|
|
|
|
<div class="name">
|
|
<strong>
|
|
Spearhead Systems Incident Response Documentation
|
|
<span class="version">
|
|
|
|
</span>
|
|
</strong>
|
|
|
|
<br>
|
|
spearheadsys/issue-response-docs
|
|
|
|
</div>
|
|
</div> -->
|
|
</a>
|
|
<div class="scrollable">
|
|
<div class="wrapper">
|
|
<!--
|
|
<ul class="repo">
|
|
<li class="repo-download">
|
|
|
|
|
|
<a href="https://github.com/spearheadsys/issue-response-docs/archive/master.zip" target="_blank" title="Download" data-action="download">
|
|
<i class="icon icon-download"></i> Download
|
|
</a>
|
|
|
|
</li>
|
|
<li class="repo-stars">
|
|
<a href="https://github.com/spearheadsys/issue-response-docs/stargazers" target="_blank" title="Stargazers" data-action="star">
|
|
<i class="icon icon-star"></i> Stars
|
|
<span class="count">–</span>
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
-->
|
|
<div class="toc">
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="Home" href="../..">
|
|
Home
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<span class="section">On-Call</span>
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="Being On-Call" href="../../oncall/being_oncall/">
|
|
Being On-Call
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Alerting Principles" href="../../oncall/alerting_principles/">
|
|
Alerting Principles
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<span class="section">Before an Incident</span>
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="Severity Levels" href="../../before/severity_levels/">
|
|
Severity Levels
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Different Roles" href="../../before/different_roles/">
|
|
Different Roles
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Call Etiquette" href="../../before/call_etiquette/">
|
|
Call Etiquette
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<span class="section">During an Incident</span>
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="During An Incident" href="../../during/during_an_incident/">
|
|
During An Incident
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Security Incident" href="../../during/security_incident_response/">
|
|
Security Incident
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<span class="section">After an Incident</span>
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="Post-Mortem Process" href="../../after/post_mortem_process/">
|
|
Post-Mortem Process
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Post-Mortem Template" href="../../after/post_mortem_template/">
|
|
Post-Mortem Template
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<span class="section">Training</span>
|
|
<ul>
|
|
|
|
|
|
<li>
|
|
<a class="" title="Overview" href="../overview/">
|
|
Overview
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Incident Commander" href="../incident_commander/">
|
|
Incident Commander
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Deputy" href="../deputy/">
|
|
Deputy
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="current" title="Scribe" href="./">
|
|
Scribe
|
|
</a>
|
|
|
|
|
|
|
|
<ul>
|
|
|
|
<li class="anchor">
|
|
<a title="Purpose" href="#purpose">
|
|
Purpose
|
|
</a>
|
|
</li>
|
|
|
|
<li class="anchor">
|
|
<a title="Prerequisites" href="#prerequisites">
|
|
Prerequisites
|
|
</a>
|
|
</li>
|
|
|
|
<li class="anchor">
|
|
<a title="Responsibilities" href="#responsibilities">
|
|
Responsibilities
|
|
</a>
|
|
</li>
|
|
|
|
<li class="anchor">
|
|
<a title="Training Process" href="#training-process">
|
|
Training Process
|
|
</a>
|
|
</li>
|
|
|
|
<li class="anchor">
|
|
<a title="Scribing" href="#scribing">
|
|
Scribing
|
|
</a>
|
|
</li>
|
|
|
|
<li class="anchor">
|
|
<a title="Incident Call Procedures and Lingo" href="#incident-call-procedures-and-lingo">
|
|
Incident Call Procedures and Lingo
|
|
</a>
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Subject Matter Expert" href="../subject_matter_expert/">
|
|
Subject Matter Expert
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="Glossary" href="../glossary/">
|
|
Glossary
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
|
|
<li>
|
|
<a class="" title="About" href="../../about/">
|
|
About
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
</div>
|
|
<article class="article">
|
|
<div class="wrapper">
|
|
|
|
<h1>Scribe</h1>
|
|
|
|
<p>So you want to be a scribe? You've come to the right place! You don't need to be a senior team member to become a deputy or scribe, anyone can do it providing you have the requisite knowledge!</p>
|
|
<p><img alt="Typewriter" src="../../assets/img/headers/typewriter.jpg" />
|
|
<em>Credit: <a href="http://www.publicdomainpictures.net/view-image.php?image=49706&picture=antique-typewriter-keys">Holly Chaffin</a></em></p>
|
|
<h2 id="purpose">Purpose<a class="headerlink" href="#purpose" title="Permanent link">#</a></h2>
|
|
<p>The purpose of the Scribe is to maintain a timeline of key events during an incident. Documenting actions, and keeping track of any followup items that will need to be addressed.</p>
|
|
<p>It's important for the rest of the command staff to be able to focus on the problem at hand, rather than worrying about documenting the steps.</p>
|
|
<p>Your job as Scribe is to listen to the call and to watch the incident Slack room, keeping track of context and actions that need to be performed, documenting these in Slack as you go. <strong>You should not be performing any remediations, checking graphs, or investigating logs.</strong> Those tasks will be delegated to the subject matter experts (SME's) by the Incident Commander.</p>
|
|
<h2 id="prerequisites">Prerequisites<a class="headerlink" href="#prerequisites" title="Permanent link">#</a></h2>
|
|
<p>Before you can be a Scribe, it is expected that you meet the following criteria. Don't worry if you don't meet them all yet, you can still continue with training!</p>
|
|
<ul>
|
|
<li>Excellent verbal and written <strong>communication skills</strong>.</li>
|
|
<li>Has <strong>knowledge of obscure PagerDuty terms</strong>.</li>
|
|
</ul>
|
|
<h2 id="responsibilities">Responsibilities<a class="headerlink" href="#responsibilities" title="Permanent link">#</a></h2>
|
|
<p>Read up on our <a href="../../before/different_roles/">Different Roles for Incidents</a> to see what is expected from a Scribe, as well as what we expect from the other roles you'll be interacting with.</p>
|
|
<h2 id="training-process">Training Process<a class="headerlink" href="#training-process" title="Permanent link">#</a></h2>
|
|
<p>There is no formal training process for this role, reading this page should be sufficient for most tasks. Here's a list of things you can do to train though,</p>
|
|
<ul>
|
|
<li>
|
|
<p>Read the rest of this page, particularly the sections below.</p>
|
|
</li>
|
|
<li>
|
|
<p>Participate in <a href="https://www.pagerduty.com/blog/failure-friday-at-pagerduty/">Failure Friday</a> (FF).</p>
|
|
<ul>
|
|
<li>Shadow a FF to see how it's run.</li>
|
|
<li>Be the scribe for multiple FF's.</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<h2 id="scribing">Scribing<a class="headerlink" href="#scribing" title="Permanent link">#</a></h2>
|
|
<p>Scribing is more art than science. The objective is to keep an accurate record of important events that occurred on the call, so that we can look back at the timeline to see what happened. But what exactly is important? There's no overwhelming answer, and it really comes down the judgement and experience. But here are some general things you most definitely want to capture as scribe.</p>
|
|
<ul>
|
|
<li>The result of any polling decisions.<ul>
|
|
<li><span class="bad">✘</span> This is not "9 people voted yay, 3 voted nay".</li>
|
|
<li><span class="good">✓</span> It is "Polled for if we should do rolling restart. <USER_A> is proceeding with restart."</li>
|
|
</ul>
|
|
</li>
|
|
<li>Any followup items that are called out as "We should do this..", "Why didn't this?..", etc.<ul>
|
|
<li><span class="bad">✘</span> This is not "Why isn't the Support representative on the call?"</li>
|
|
<li><span class="good">✓</span> This is "TODO: Why didn't we get paged for this earlier?"</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<h2 id="incident-call-procedures-and-lingo">Incident Call Procedures and Lingo<a class="headerlink" href="#incident-call-procedures-and-lingo" title="Permanent link">#</a></h2>
|
|
<p>The <a href="../../during/during_an_incident/">Steps for Scribe</a> provide a detailed description of what you should be doing during an incident.</p>
|
|
<p>Here are some examples of phrases and patterns you should use during incident calls.</p>
|
|
<h3 id="status-stalking">Status Stalking<a class="headerlink" href="#status-stalking" title="Permanent link">#</a></h3>
|
|
<p>At the start of any major incident call, you should start our status stalking bot, so that it will post to the room an update automatically.</p>
|
|
<blockquote>
|
|
<p>!status stalk</p>
|
|
</blockquote>
|
|
<p>This will provide the update and allow the IC to see the status without having to keep asking.</p>
|
|
<h3 id="note-important-actions">Note Important Actions<a class="headerlink" href="#note-important-actions" title="Permanent link">#</a></h3>
|
|
<p>During a call, you will hear lots of discussion happening, you should not be documenting all of this in the chat room. You only want to document things which will be important for the final timeline. It's not always obvious what this might be, and it's usually a matter of judgement. You generally want to note any actions the IC has asked someone to perform, along with the result of any polling decisions.</p>
|
|
<blockquote>
|
|
<p>Polled for decision on whether to perform rolling restart. We are proceeding with restart. [USER_A] to execute.</p>
|
|
</blockquote>
|
|
<p>Some actions might seem important at the time, but end up not being. That's OK. It's better to have more info than not enough, but don't go overboard.</p>
|
|
<h3 id="note-followup-actions">Note Followup Actions<a class="headerlink" href="#note-followup-actions" title="Permanent link">#</a></h3>
|
|
<p>Sometimes during the call, someone will either mention something we "should fix", or the IC will specifically ask you to note a followup item. You can do this in Slack by simply prefixing with "TODO", this will make it easier to search for later.</p>
|
|
<blockquote>
|
|
<p>TODO: Why did we not get paged for the fall in traffic on [X] cluster?</p>
|
|
</blockquote>
|
|
<p>The post-mortem owner will find these after and raise tasks for them.</p>
|
|
<h3 id="end-of-call-notification">End of Call Notification<a class="headerlink" href="#end-of-call-notification" title="Permanent link">#</a></h3>
|
|
<p>When the IC ends the call, you should post a message into Slack to let everyone know the call is over, and that they should continue discussion elsewhere.</p>
|
|
<blockquote>
|
|
<p>Call is over, thanks everyone. Follow up in Slack.</p>
|
|
</blockquote>
|
|
<p>Don't forget to also stop the status stalking.</p>
|
|
<blockquote>
|
|
<p>!status unstalk</p>
|
|
</blockquote>
|
|
<aside class="copyright" role="note">
|
|
|
|
Copyright © Spearhead Systems, Inc. –
|
|
|
|
Documentation built with
|
|
<a href="http://www.mkdocs.org" target="_blank">MkDocs</a>
|
|
using the
|
|
<a href="http://squidfunk.github.io/mkdocs-material/" target="_blank">
|
|
Material
|
|
</a>
|
|
theme.
|
|
</aside>
|
|
|
|
<footer class="footer">
|
|
|
|
<nav class="pagination" aria-label="Footer">
|
|
<div class="previous">
|
|
|
|
<a href="../deputy/" title="Deputy">
|
|
<span class="direction">
|
|
Previous
|
|
</span>
|
|
<div class="page">
|
|
<div class="button button-previous" role="button" aria-label="Previous">
|
|
<i class="icon icon-back"></i>
|
|
</div>
|
|
<div class="stretch">
|
|
<div class="title">
|
|
Deputy
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</a>
|
|
|
|
</div>
|
|
<div class="next">
|
|
|
|
<a href="../subject_matter_expert/" title="Subject Matter Expert">
|
|
<span class="direction">
|
|
Next
|
|
</span>
|
|
<div class="page">
|
|
<div class="stretch">
|
|
<div class="title">
|
|
Subject Matter Expert
|
|
</div>
|
|
</div>
|
|
<div class="button button-next" role="button" aria-label="Next">
|
|
<i class="icon icon-forward"></i>
|
|
</div>
|
|
</div>
|
|
</a>
|
|
|
|
</div>
|
|
</nav>
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</article>
|
|
<div class="results" role="status" aria-live="polite">
|
|
<div class="scrollable">
|
|
<div class="wrapper">
|
|
<div class="meta"></div>
|
|
<div class="list"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</main>
|
|
<script>
|
|
var base_url = '../..';
|
|
var repo_id = 'spearheadsys/issue-response-docs';
|
|
</script>
|
|
<script src="../../assets/javascripts/application-997097ee0c.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |