\hypertarget{kern__sx_8c_source}{}\doxysection{kern\+\_\+sx.\+c}
\label{kern__sx_8c_source}\index{kern/kern\_sx.c@{kern/kern\_sx.c}}
\mbox{\hyperlink{kern__sx_8c}{Go to the documentation of this file.}}
\begin{DoxyCode}{0}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00001}00001 \textcolor{comment}{/*-\/}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00002}00002 \textcolor{comment}{ * SPDX-\/License-\/Identifier: BSD-\/2-\/Clause-\/FreeBSD}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00003}00003 \textcolor{comment}{ *}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00004}00004 \textcolor{comment}{ * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00005}00005 \textcolor{comment}{ * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00006}00006 \textcolor{comment}{ * All rights reserved.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00007}00007 \textcolor{comment}{ *}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00008}00008 \textcolor{comment}{ * Redistribution and use in source and binary forms, with or without}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00009}00009 \textcolor{comment}{ * modification, are permitted provided that the following conditions}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00010}00010 \textcolor{comment}{ * are met:}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00011}00011 \textcolor{comment}{ * 1. Redistributions of source code must retain the above copyright}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00012}00012 \textcolor{comment}{ *    notice(s), this list of conditions and the following disclaimer as}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00013}00013 \textcolor{comment}{ *    the first lines of this file unmodified other than the possible}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00014}00014 \textcolor{comment}{ *    addition of one or more copyright notices.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00015}00015 \textcolor{comment}{ * 2. Redistributions in binary form must reproduce the above copyright}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00016}00016 \textcolor{comment}{ *    notice(s), this list of conditions and the following disclaimer in the}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00017}00017 \textcolor{comment}{ *    documentation and/or other materials provided with the distribution.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00018}00018 \textcolor{comment}{ *}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00019}00019 \textcolor{comment}{ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00020}00020 \textcolor{comment}{ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00021}00021 \textcolor{comment}{ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00022}00022 \textcolor{comment}{ * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00023}00023 \textcolor{comment}{ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00024}00024 \textcolor{comment}{ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00025}00025 \textcolor{comment}{ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00026}00026 \textcolor{comment}{ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00027}00027 \textcolor{comment}{ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00028}00028 \textcolor{comment}{ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00029}00029 \textcolor{comment}{ * DAMAGE.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00030}00030 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00031}00031 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00032}00032 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00033}00033 \textcolor{comment}{ * Shared/exclusive locks.  This implementation attempts to ensure}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00034}00034 \textcolor{comment}{ * deterministic lock granting behavior, so that slocks and xlocks are}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00035}00035 \textcolor{comment}{ * interleaved.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00036}00036 \textcolor{comment}{ *}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00037}00037 \textcolor{comment}{ * Priority propagation will not generally raise the priority of lock holders,}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00038}00038 \textcolor{comment}{ * so should not be relied upon in combination with sx locks.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00039}00039 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00040}00040 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00041}00041 \textcolor{preprocessor}{\#include "{}opt\_ddb.h"{}}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00042}00042 \textcolor{preprocessor}{\#include "{}opt\_hwpmc\_hooks.h"{}}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00043}00043 \textcolor{preprocessor}{\#include "{}opt\_no\_adaptive\_sx.h"{}}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00044}00044 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00045}00045 \textcolor{preprocessor}{\#include <sys/cdefs.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00046}\mbox{\hyperlink{kern__sx_8c_a8c3ef283470a68cd30afe9b8a72cb60f}{00046}} \mbox{\hyperlink{kern__sx_8c_a8c3ef283470a68cd30afe9b8a72cb60f}{\_\_FBSDID}}(\textcolor{stringliteral}{"{}\$FreeBSD\$"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00047}00047 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00048}00048 \textcolor{preprocessor}{\#include <sys/param.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00049}00049 \textcolor{preprocessor}{\#include <sys/systm.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00050}00050 \textcolor{preprocessor}{\#include <sys/kdb.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00051}00051 \textcolor{preprocessor}{\#include <sys/kernel.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00052}00052 \textcolor{preprocessor}{\#include <sys/ktr.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00053}00053 \textcolor{preprocessor}{\#include <sys/lock.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00054}00054 \textcolor{preprocessor}{\#include <sys/mutex.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00055}00055 \textcolor{preprocessor}{\#include <sys/proc.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00056}00056 \textcolor{preprocessor}{\#include <sys/sched.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00057}00057 \textcolor{preprocessor}{\#include <sys/sleepqueue.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00058}00058 \textcolor{preprocessor}{\#include <sys/sx.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00059}00059 \textcolor{preprocessor}{\#include <sys/smp.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00060}00060 \textcolor{preprocessor}{\#include <sys/sysctl.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00061}00061 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00062}00062 \textcolor{preprocessor}{\#if defined(SMP) \&\& !defined(NO\_ADAPTIVE\_SX)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00063}00063 \textcolor{preprocessor}{\#include <machine/cpu.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00064}00064 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00065}00065 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00066}00066 \textcolor{preprocessor}{\#ifdef DDB}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00067}00067 \textcolor{preprocessor}{\#include <ddb/ddb.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00068}00068 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00069}00069 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00070}00070 \textcolor{preprocessor}{\#if defined(SMP) \&\& !defined(NO\_ADAPTIVE\_SX)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00071}00071 \textcolor{preprocessor}{\#define ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00072}00072 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00073}00073 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00074}00074 \textcolor{preprocessor}{\#ifdef HWPMC\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00075}00075 \textcolor{preprocessor}{\#include <sys/pmckern.h>}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00076}00076 PMC\_SOFT\_DECLARE( , , lock, failed);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00077}00077 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00078}00078 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00079}00079 \textcolor{comment}{/* Handy macros for sleep queues. */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00080}\mbox{\hyperlink{kern__sx_8c_a3cde1e96adc94f05ec2d174559d9cb8d}{00080}} \textcolor{preprocessor}{\#define SQ\_EXCLUSIVE\_QUEUE      0}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00081}\mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{00081}} \textcolor{preprocessor}{\#define SQ\_SHARED\_QUEUE         1}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00082}00082 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00083}00083 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00084}00084 \textcolor{comment}{ * Variations on DROP\_GIANT()/PICKUP\_GIANT() for use in this file.  We}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00085}00085 \textcolor{comment}{ * drop Giant anytime we have to sleep or if we adaptively spin.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00086}00086 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00087}\mbox{\hyperlink{kern__sx_8c_a9cc3c95f2349916c70a759a449314da5}{00087}} \textcolor{preprocessor}{\#define GIANT\_DECLARE                                                   \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00088}00088 \textcolor{preprocessor}{        int \_giantcnt = 0;                                              \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00089}00089 \textcolor{preprocessor}{        WITNESS\_SAVE\_DECL(Giant)                                        \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00090}00090 \textcolor{preprocessor}{}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00091}\mbox{\hyperlink{kern__sx_8c_a0f4702099e1ba4bb91339c5e1139caa0}{00091}} \textcolor{preprocessor}{\#define GIANT\_SAVE(work) do \{                                           \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00092}00092 \textcolor{preprocessor}{        if (\_\_predict\_false(mtx\_owned(\&Giant))) \{                       \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00093}00093 \textcolor{preprocessor}{                work++;                                                 \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00094}00094 \textcolor{preprocessor}{                WITNESS\_SAVE(\&Giant.lock\_object, Giant);                \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00095}00095 \textcolor{preprocessor}{                while (mtx\_owned(\&Giant)) \{                             \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00096}00096 \textcolor{preprocessor}{                        \_giantcnt++;                                    \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00097}00097 \textcolor{preprocessor}{                        mtx\_unlock(\&Giant);                             \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00098}00098 \textcolor{preprocessor}{                \}                                                       \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00099}00099 \textcolor{preprocessor}{        \}                                                               \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00100}00100 \textcolor{preprocessor}{\} while (0)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00101}00101 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00102}\mbox{\hyperlink{kern__sx_8c_a4071a8f3605a24fcdac5e4cba9d0526f}{00102}} \textcolor{preprocessor}{\#define GIANT\_RESTORE() do \{                                            \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00103}00103 \textcolor{preprocessor}{        if (\_giantcnt > 0) \{                                            \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00104}00104 \textcolor{preprocessor}{                mtx\_assert(\&Giant, MA\_NOTOWNED);                        \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00105}00105 \textcolor{preprocessor}{                while (\_giantcnt-\/-\/)                                     \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00106}00106 \textcolor{preprocessor}{                        mtx\_lock(\&Giant);                               \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00107}00107 \textcolor{preprocessor}{                WITNESS\_RESTORE(\&Giant.lock\_object, Giant);             \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00108}00108 \textcolor{preprocessor}{        \}                                                               \(\backslash\)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00109}00109 \textcolor{preprocessor}{\} while (0)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00110}00110 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00111}00111 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00112}00112 \textcolor{comment}{ * Returns true if an exclusive lock is recursed.  It assumes}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00113}00113 \textcolor{comment}{ * curthread currently has an exclusive lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00114}00114 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00115}\mbox{\hyperlink{kern__sx_8c_a5745cf1e8f892977266d7fd5d04dbe2b}{00115}} \textcolor{preprocessor}{\#define sx\_recursed(sx)         ((sx)-\/>sx\_recurse != 0)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00116}00116 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00117}00117 \textcolor{keyword}{static} \textcolor{keywordtype}{void}     \mbox{\hyperlink{kern__sx_8c_a126c78d982bd70dba13096237fec863c}{assert\_sx}}(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock, \textcolor{keywordtype}{int} what);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00118}00118 \textcolor{preprocessor}{\#ifdef DDB}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00119}00119 \textcolor{keyword}{static} \textcolor{keywordtype}{void}     db\_show\_sx(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00120}00120 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00121}00121 \textcolor{keyword}{static} \textcolor{keywordtype}{void}     \mbox{\hyperlink{kern__sx_8c_a1ec0c1b36987d7a629af9a5d3483233b}{lock\_sx}}(\textcolor{keyword}{struct} lock\_object *lock, uintptr\_t how);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00122}00122 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00123}00123 \textcolor{keyword}{static} \textcolor{keywordtype}{int}      owner\_sx(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock, \textcolor{keyword}{struct} thread **owner);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00124}00124 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00125}00125 \textcolor{keyword}{static} uintptr\_t \mbox{\hyperlink{kern__sx_8c_ae2a2d710b2b973aae2634f73b062a151}{unlock\_sx}}(\textcolor{keyword}{struct} lock\_object *lock);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00126}00126 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00127}\mbox{\hyperlink{kern__sx_8c_a188613c956d39c9b944b9de278ce2801}{00127}} \textcolor{keyword}{struct }lock\_class \mbox{\hyperlink{kern__sx_8c_a188613c956d39c9b944b9de278ce2801}{lock\_class\_sx}} = \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00128}00128         .lc\_name = \textcolor{stringliteral}{"{}sx"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00129}00129         .lc\_flags = LC\_SLEEPLOCK | LC\_SLEEPABLE | LC\_RECURSABLE | LC\_UPGRADABLE,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00130}00130         .lc\_assert = \mbox{\hyperlink{kern__sx_8c_a126c78d982bd70dba13096237fec863c}{assert\_sx}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00131}00131 \textcolor{preprocessor}{\#ifdef DDB}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00132}00132         .lc\_ddb\_show = db\_show\_sx,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00133}00133 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00134}00134         .lc\_lock = \mbox{\hyperlink{kern__sx_8c_a1ec0c1b36987d7a629af9a5d3483233b}{lock\_sx}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00135}00135         .lc\_unlock = \mbox{\hyperlink{kern__sx_8c_ae2a2d710b2b973aae2634f73b062a151}{unlock\_sx}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00136}00136 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00137}00137         .lc\_owner = owner\_sx,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00138}00138 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00139}00139 \};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00140}00140 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00141}00141 \textcolor{preprocessor}{\#ifndef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00142}\mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{00142}} \textcolor{preprocessor}{\#define \_sx\_assert(sx, what, file, line)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00143}00143 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00144}00144 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00145}00145 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00146}00146 \textcolor{preprocessor}{\#ifdef SX\_CUSTOM\_BACKOFF}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00147}00147 \textcolor{keyword}{static} u\_short \_\_read\_frequently asx\_retries;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00148}00148 \textcolor{keyword}{static} u\_short \_\_read\_frequently asx\_loops;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00149}00149 \textcolor{keyword}{static} \mbox{\hyperlink{imgact__binmisc_8c_a745707e006bc5c9b6f81b78dfc5caadd}{SYSCTL\_NODE}}(\_debug, OID\_AUTO, sx, CTLFLAG\_RD | CTLFLAG\_MPSAFE, NULL,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00150}00150     \textcolor{stringliteral}{"{}sxlock debugging"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00151}00151 \mbox{\hyperlink{subr__lock_8c_a104546c64100ef334c7f6273e574c4c6}{SYSCTL\_U16}}(\_debug\_sx, OID\_AUTO, retries, CTLFLAG\_RW, \&asx\_retries, 0, \textcolor{stringliteral}{"{}"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00152}00152 \mbox{\hyperlink{subr__lock_8c_a104546c64100ef334c7f6273e574c4c6}{SYSCTL\_U16}}(\_debug\_sx, OID\_AUTO, loops, CTLFLAG\_RW, \&asx\_loops, 0, \textcolor{stringliteral}{"{}"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00153}00153 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00154}00154 \textcolor{keyword}{static} \textcolor{keyword}{struct }lock\_delay\_config \_\_read\_frequently sx\_delay;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00155}00155 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00156}00156 \mbox{\hyperlink{subr__lock_8c_a104546c64100ef334c7f6273e574c4c6}{SYSCTL\_U16}}(\_debug\_sx, OID\_AUTO, delay\_base, CTLFLAG\_RW, \&sx\_delay.base,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00157}00157     0, \textcolor{stringliteral}{"{}"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00158}00158 \mbox{\hyperlink{subr__lock_8c_a104546c64100ef334c7f6273e574c4c6}{SYSCTL\_U16}}(\_debug\_sx, OID\_AUTO, delay\_max, CTLFLAG\_RW, \&sx\_delay.max,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00159}00159     0, \textcolor{stringliteral}{"{}"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00160}00160 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00161}00161 \textcolor{keyword}{static} \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00162}00162 sx\_lock\_delay\_init(\textcolor{keywordtype}{void} *arg \_\_unused)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00163}00163 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00164}00164 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00165}00165         \mbox{\hyperlink{subr__lock_8c_a1995ff0720f9abe78fa65ad9e124f09b}{lock\_delay\_default\_init}}(\&sx\_delay);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00166}00166         asx\_retries = 10;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00167}00167         asx\_loops = max(10000, sx\_delay.max);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00168}00168 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00169}00169 \mbox{\hyperlink{subr__lock_8c_ab1d35deed274d2f1dbc71d7b9e281597}{LOCK\_DELAY\_SYSINIT}}(sx\_lock\_delay\_init);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00170}00170 \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00171}00171 \textcolor{preprocessor}{\#define sx\_delay        locks\_delay}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00172}00172 \textcolor{preprocessor}{\#define asx\_retries     locks\_delay\_retries}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00173}00173 \textcolor{preprocessor}{\#define asx\_loops       locks\_delay\_loops}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00174}00174 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00175}00175 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00176}00176 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00177}00177 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00178}\mbox{\hyperlink{kern__sx_8c_a126c78d982bd70dba13096237fec863c}{00178}} \mbox{\hyperlink{kern__sx_8c_a126c78d982bd70dba13096237fec863c}{assert\_sx}}(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock, \textcolor{keywordtype}{int} what)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00179}00179 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00180}00180 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00181}00181         sx\_assert((\textcolor{keyword}{const} \textcolor{keyword}{struct} sx *)lock, what);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00182}00182 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00183}00183 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00184}00184 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00185}\mbox{\hyperlink{kern__sx_8c_a1ec0c1b36987d7a629af9a5d3483233b}{00185}} \mbox{\hyperlink{kern__sx_8c_a1ec0c1b36987d7a629af9a5d3483233b}{lock\_sx}}(\textcolor{keyword}{struct} lock\_object *lock, uintptr\_t how)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00186}00186 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00187}00187         \textcolor{keyword}{struct }sx *sx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00188}00188 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00189}00189         sx = (\textcolor{keyword}{struct }sx *)lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00190}00190         \textcolor{keywordflow}{if} (how)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00191}00191                 sx\_slock(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00192}00192         \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00193}00193                 sx\_xlock(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00194}00194 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00195}00195 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00196}00196 uintptr\_t}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00197}\mbox{\hyperlink{kern__sx_8c_ae2a2d710b2b973aae2634f73b062a151}{00197}} \mbox{\hyperlink{kern__sx_8c_ae2a2d710b2b973aae2634f73b062a151}{unlock\_sx}}(\textcolor{keyword}{struct} lock\_object *lock)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00198}00198 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00199}00199         \textcolor{keyword}{struct }sx *sx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00200}00200 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00201}00201         sx = (\textcolor{keyword}{struct }sx *)lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00202}00202         sx\_assert(sx, SA\_LOCKED | SA\_NOTRECURSED);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00203}00203         \textcolor{keywordflow}{if} (sx\_xlocked(sx)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00204}00204                 sx\_xunlock(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00205}00205                 \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00206}00206         \} \textcolor{keywordflow}{else} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00207}00207                 sx\_sunlock(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00208}00208                 \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00209}00209         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00210}00210 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00211}00211 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00212}00212 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00213}00213 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00214}00214 owner\_sx(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock, \textcolor{keyword}{struct} thread **owner)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00215}00215 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00216}00216         \textcolor{keyword}{const} \textcolor{keyword}{struct }sx *sx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00217}00217         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00218}00218 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00219}00219         sx = (\textcolor{keyword}{const} \textcolor{keyword}{struct }sx *)lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00220}00220         x = sx-\/>sx\_lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00221}00221         *owner = NULL;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00222}00222         \textcolor{keywordflow}{return} ((x \& SX\_LOCK\_SHARED) != 0 ? (SX\_SHARERS(x) != 0) :}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00223}00223             ((*owner = (\textcolor{keyword}{struct} thread *)SX\_OWNER(x)) != NULL));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00224}00224 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00225}00225 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00226}00226 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00227}00227 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00228}\mbox{\hyperlink{kern__sx_8c_a1e5c28ce122e977d9ef3b4e7b90c57a1}{00228}} \mbox{\hyperlink{kern__sx_8c_a1e5c28ce122e977d9ef3b4e7b90c57a1}{sx\_sysinit}}(\textcolor{keywordtype}{void} *arg)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00229}00229 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00230}00230         \textcolor{keyword}{struct }sx\_args *sargs = arg;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00231}00231 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00232}00232         \mbox{\hyperlink{kern__sx_8c_a229d232dd752a0cc68ecb23d7e6e75ab}{sx\_init\_flags}}(sargs-\/>sa\_sx, sargs-\/>sa\_desc, sargs-\/>sa\_flags);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00233}00233 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00234}00234 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00235}00235 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00236}\mbox{\hyperlink{kern__sx_8c_a229d232dd752a0cc68ecb23d7e6e75ab}{00236}} \mbox{\hyperlink{kern__sx_8c_a229d232dd752a0cc68ecb23d7e6e75ab}{sx\_init\_flags}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *description, \textcolor{keywordtype}{int} opts)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00237}00237 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00238}00238         \textcolor{keywordtype}{int} \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00239}00239 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00240}00240         MPASS((opts \& \string~(SX\_QUIET | SX\_RECURSE | SX\_NOWITNESS | SX\_DUPOK |}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00241}00241             SX\_NOPROFILE | SX\_NEW)) == 0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00242}00242         ASSERT\_ATOMIC\_LOAD\_PTR(sx-\/>sx\_lock,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00243}00243             (\textcolor{stringliteral}{"{}\%s: sx\_lock not aligned for \%s: \%p"{}}, \_\_func\_\_, description,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00244}00244             \&sx-\/>sx\_lock));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00245}00245 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00246}00246         \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} = LO\_SLEEPABLE | LO\_UPGRADABLE;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00247}00247         \textcolor{keywordflow}{if} (opts \& SX\_DUPOK)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00248}00248                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_DUPOK;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00249}00249         \textcolor{keywordflow}{if} (opts \& SX\_NOPROFILE)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00250}00250                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_NOPROFILE;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00251}00251         \textcolor{keywordflow}{if} (!(opts \& SX\_NOWITNESS))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00252}00252                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_WITNESS;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00253}00253         \textcolor{keywordflow}{if} (opts \& SX\_RECURSE)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00254}00254                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_RECURSABLE;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00255}00255         \textcolor{keywordflow}{if} (opts \& SX\_QUIET)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00256}00256                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_QUIET;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00257}00257         \textcolor{keywordflow}{if} (opts \& SX\_NEW)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00258}00258                 \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}} |= LO\_NEW;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00259}00259 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00260}00260         \mbox{\hyperlink{subr__lock_8c_a8a5797274e8546fc851fbc9579aae8d3}{lock\_init}}(\&sx-\/>lock\_object, \&\mbox{\hyperlink{kern__sx_8c_a188613c956d39c9b944b9de278ce2801}{lock\_class\_sx}}, description, NULL, \mbox{\hyperlink{subr__stats_8c_a1e87af3c18a2fd36c61faf89949bdc3f}{flags}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00261}00261         sx-\/>sx\_lock = SX\_LOCK\_UNLOCKED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00262}00262         sx-\/>sx\_recurse = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00263}00263 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00264}00264 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00265}00265 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00266}\mbox{\hyperlink{kern__sx_8c_a7fdb7b1b5a911e5998ce226e9532adf1}{00266}} \mbox{\hyperlink{kern__sx_8c_a7fdb7b1b5a911e5998ce226e9532adf1}{sx\_destroy}}(\textcolor{keyword}{struct} sx *sx)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00267}00267 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00268}00268 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00269}00269         KASSERT(sx-\/>sx\_lock == SX\_LOCK\_UNLOCKED, (\textcolor{stringliteral}{"{}sx lock still held"{}}));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00270}00270         KASSERT(sx-\/>sx\_recurse == 0, (\textcolor{stringliteral}{"{}sx lock still recursed"{}}));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00271}00271         sx-\/>sx\_lock = SX\_LOCK\_DESTROYED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00272}00272         \mbox{\hyperlink{subr__lock_8c_a2b0d38f10a5ed4e30f82f6537ffc9800}{lock\_destroy}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00273}00273 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00274}00274 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00275}00275 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00276}\mbox{\hyperlink{kern__sx_8c_a0a551f1ea2fe63ce69197c72240490ac}{00276}} \mbox{\hyperlink{kern__sx_8c_a0a551f1ea2fe63ce69197c72240490ac}{sx\_try\_slock\_int}}(\textcolor{keyword}{struct} sx *sx LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00277}00277 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00278}00278         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00279}00279 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00280}00280         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00281}00281                 \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00282}00282 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00283}00283         KASSERT(\mbox{\hyperlink{subr__kdb_8c_a313e8ff6f20d39fd246ef5da5648c08c}{kdb\_active}} != 0 || !TD\_IS\_IDLETHREAD(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00284}00284             (\textcolor{stringliteral}{"{}sx\_try\_slock() by idle thread \%p on sx \%s @ \%s:\%d"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00285}00285             curthread, sx-\/>lock\_object.lo\_name, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00286}00286 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00287}00287         x = sx-\/>sx\_lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00288}00288         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00289}00289                 KASSERT(x != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00290}00290                     (\textcolor{stringliteral}{"{}sx\_try\_slock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00291}00291                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00292}00292                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00293}00293                 \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, x + SX\_ONE\_SHARER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00294}00294                         LOCK\_LOG\_TRY(\textcolor{stringliteral}{"{}SLOCK"{}}, \&sx-\/>lock\_object, 0, 1, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00295}00295                         WITNESS\_LOCK(\&sx-\/>lock\_object, LOP\_TRYLOCK, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00296}00296                         LOCKSTAT\_PROFILE\_OBTAIN\_RWLOCK\_SUCCESS(sx\_\_acquire,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00297}00297                             sx, 0, 0, file, line, LOCKSTAT\_READER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00298}00298                         TD\_LOCKS\_INC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00299}00299                         curthread-\/>td\_sx\_slocks++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00300}00300                         \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00301}00301                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00302}00302         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00303}00303 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00304}00304         LOCK\_LOG\_TRY(\textcolor{stringliteral}{"{}SLOCK"{}}, \&sx-\/>lock\_object, 0, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00305}00305         \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00306}00306 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00307}00307 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00308}00308 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00309}\mbox{\hyperlink{kern__sx_8c_a425bba198755b8c06555c608f8d40760}{00309}} \mbox{\hyperlink{kern__sx_8c_a425bba198755b8c06555c608f8d40760}{sx\_try\_slock\_}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00310}00310 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00311}00311 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00312}00312         \textcolor{keywordflow}{return} (\mbox{\hyperlink{kern__sx_8c_a0a551f1ea2fe63ce69197c72240490ac}{sx\_try\_slock\_int}}(sx LOCK\_FILE\_LINE\_ARG));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00313}00313 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00314}00314 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00315}00315 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00316}\mbox{\hyperlink{kern__sx_8c_afd5b60d873c35da597c0a18cd2310faf}{00316}} \mbox{\hyperlink{kern__sx_8c_afd5b60d873c35da597c0a18cd2310faf}{\_sx\_xlock}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keywordtype}{int} opts, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00317}00317 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00318}00318         uintptr\_t tid, x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00319}00319         \textcolor{keywordtype}{int} error = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00320}00320 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00321}00321         KASSERT(\mbox{\hyperlink{subr__kdb_8c_a313e8ff6f20d39fd246ef5da5648c08c}{kdb\_active}} != 0 || SCHEDULER\_STOPPED() ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00322}00322             !TD\_IS\_IDLETHREAD(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00323}00323             (\textcolor{stringliteral}{"{}sx\_xlock() by idle thread \%p on sx \%s @ \%s:\%d"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00324}00324             curthread, sx-\/>lock\_object.lo\_name, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00325}00325         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00326}00326             (\textcolor{stringliteral}{"{}sx\_xlock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00327}00327         WITNESS\_CHECKORDER(\&sx-\/>lock\_object, LOP\_NEWORDER | LOP\_EXCLUSIVE, file,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00328}00328             line, NULL);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00329}00329         tid = (uintptr\_t)curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00330}00330         x = SX\_LOCK\_UNLOCKED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00331}00331         \textcolor{keywordflow}{if} (!atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00332}00332                 error = \mbox{\hyperlink{kern__sx_8c_acc52958e610760d9e978ecdb14365bc9}{\_sx\_xlock\_hard}}(sx, x, opts LOCK\_FILE\_LINE\_ARG);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00333}00333         \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00334}00334                 LOCKSTAT\_PROFILE\_OBTAIN\_RWLOCK\_SUCCESS(sx\_\_acquire, sx,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00335}00335                     0, 0, file, line, LOCKSTAT\_WRITER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00336}00336         \textcolor{keywordflow}{if} (!error) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00337}00337                 LOCK\_LOG\_LOCK(\textcolor{stringliteral}{"{}XLOCK"{}}, \&sx-\/>lock\_object, 0, sx-\/>sx\_recurse,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00338}00338                     file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00339}00339                 WITNESS\_LOCK(\&sx-\/>lock\_object, LOP\_EXCLUSIVE, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00340}00340                 TD\_LOCKS\_INC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00341}00341         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00342}00342 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00343}00343         \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00344}00344 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00345}00345 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00346}00346 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00347}\mbox{\hyperlink{kern__sx_8c_aa02d82d982788408c448ea118ee15d59}{00347}} \mbox{\hyperlink{kern__sx_8c_aa02d82d982788408c448ea118ee15d59}{sx\_try\_xlock\_int}}(\textcolor{keyword}{struct} sx *sx LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00348}00348 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00349}00349         \textcolor{keyword}{struct }thread *td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00350}00350         uintptr\_t tid, x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00351}00351         \textcolor{keywordtype}{int} rval;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00352}00352         \textcolor{keywordtype}{bool} recursed;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00353}00353 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00354}00354         td = curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00355}00355         tid = (uintptr\_t)td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00356}00356         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED\_TD(td))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00357}00357                 \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00358}00358 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00359}00359         KASSERT(\mbox{\hyperlink{subr__kdb_8c_a313e8ff6f20d39fd246ef5da5648c08c}{kdb\_active}} != 0 || !TD\_IS\_IDLETHREAD(td),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00360}00360             (\textcolor{stringliteral}{"{}sx\_try\_xlock() by idle thread \%p on sx \%s @ \%s:\%d"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00361}00361             curthread, sx-\/>lock\_object.lo\_name, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00362}00362         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00363}00363             (\textcolor{stringliteral}{"{}sx\_try\_xlock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00364}00364 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00365}00365         rval = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00366}00366         recursed = \textcolor{keyword}{false};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00367}00367         x = SX\_LOCK\_UNLOCKED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00368}00368         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00369}00369                 \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00370}00370                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00371}00371                 \textcolor{keywordflow}{if} (x == SX\_LOCK\_UNLOCKED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00372}00372                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00373}00373                 \textcolor{keywordflow}{if} (x == tid \&\& (sx-\/>lock\_object.lo\_flags \& LO\_RECURSABLE)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00374}00374                         sx-\/>sx\_recurse++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00375}00375                         atomic\_set\_ptr(\&sx-\/>sx\_lock, SX\_LOCK\_RECURSED);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00376}00376                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00377}00377                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00378}00378                 rval = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00379}00379                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00380}00380         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00381}00381 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00382}00382         LOCK\_LOG\_TRY(\textcolor{stringliteral}{"{}XLOCK"{}}, \&sx-\/>lock\_object, 0, rval, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00383}00383         \textcolor{keywordflow}{if} (rval) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00384}00384                 WITNESS\_LOCK(\&sx-\/>lock\_object, LOP\_EXCLUSIVE | LOP\_TRYLOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00385}00385                     file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00386}00386                 \textcolor{keywordflow}{if} (!recursed)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00387}00387                         LOCKSTAT\_PROFILE\_OBTAIN\_RWLOCK\_SUCCESS(sx\_\_acquire,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00388}00388                             sx, 0, 0, file, line, LOCKSTAT\_WRITER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00389}00389                 TD\_LOCKS\_INC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00390}00390         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00391}00391 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00392}00392         \textcolor{keywordflow}{return} (rval);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00393}00393 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00394}00394 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00395}00395 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00396}\mbox{\hyperlink{kern__sx_8c_aa7726de1ed82c4a88b8c4462a3711989}{00396}} \mbox{\hyperlink{kern__sx_8c_aa7726de1ed82c4a88b8c4462a3711989}{sx\_try\_xlock\_}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00397}00397 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00398}00398 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00399}00399         \textcolor{keywordflow}{return} (\mbox{\hyperlink{kern__sx_8c_aa02d82d982788408c448ea118ee15d59}{sx\_try\_xlock\_int}}(sx LOCK\_FILE\_LINE\_ARG));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00400}00400 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00401}00401 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00402}00402 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00403}\mbox{\hyperlink{kern__sx_8c_a4c442b0adeb98ff4b1dc1d3fe5b1d126}{00403}} \mbox{\hyperlink{kern__sx_8c_a4c442b0adeb98ff4b1dc1d3fe5b1d126}{\_sx\_xunlock}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00404}00404 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00405}00405 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00406}00406         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00407}00407             (\textcolor{stringliteral}{"{}sx\_xunlock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00408}00408         \mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{\_sx\_assert}}(sx, SA\_XLOCKED, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00409}00409         WITNESS\_UNLOCK(\&sx-\/>lock\_object, LOP\_EXCLUSIVE, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00410}00410         LOCK\_LOG\_LOCK(\textcolor{stringliteral}{"{}XUNLOCK"{}}, \&sx-\/>lock\_object, 0, sx-\/>sx\_recurse, file,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00411}00411             line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00412}00412 \textcolor{preprocessor}{\#if LOCK\_DEBUG > 0}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00413}00413         \mbox{\hyperlink{kern__sx_8c_a038866f68f0a49227c3ec51014e077d3}{\_sx\_xunlock\_hard}}(sx, (uintptr\_t)curthread, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00414}00414 \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00415}00415         \_\_sx\_xunlock(sx, curthread, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00416}00416 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00417}00417         TD\_LOCKS\_DEC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00418}00418 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00419}00419 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00420}00420 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00421}00421 \textcolor{comment}{ * Try to do a non-\/blocking upgrade from a shared lock to an exclusive lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00422}00422 \textcolor{comment}{ * This will only succeed if this thread holds a single shared lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00423}00423 \textcolor{comment}{ * Return 1 if if the upgrade succeed, 0 otherwise.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00424}00424 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00425}00425 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00426}\mbox{\hyperlink{kern__sx_8c_a39f6a5d89e9fb03bc6cc567d185e7143}{00426}} \mbox{\hyperlink{kern__sx_8c_a39f6a5d89e9fb03bc6cc567d185e7143}{sx\_try\_upgrade\_int}}(\textcolor{keyword}{struct} sx *sx LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00427}00427 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00428}00428         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00429}00429         uintptr\_t waiters;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00430}00430         \textcolor{keywordtype}{int} success;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00431}00431 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00432}00432         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00433}00433                 \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00434}00434 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00435}00435         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00436}00436             (\textcolor{stringliteral}{"{}sx\_try\_upgrade() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00437}00437         \mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{\_sx\_assert}}(sx, SA\_SLOCKED, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00438}00438 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00439}00439         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00440}00440 \textcolor{comment}{         * Try to switch from one shared lock to an exclusive lock.  We need}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00441}00441 \textcolor{comment}{         * to maintain the SX\_LOCK\_EXCLUSIVE\_WAITERS flag if set so that}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00442}00442 \textcolor{comment}{         * we will wake up the exclusive waiters when we drop the lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00443}00443 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00444}00444         success = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00445}00445         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00446}00446         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00447}00447                 \textcolor{keywordflow}{if} (SX\_SHARERS(x) > 1)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00448}00448                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00449}00449                 waiters = (x \& SX\_LOCK\_WAITERS);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00450}00450                 \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00451}00451                     (uintptr\_t)curthread | waiters)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00452}00452                         success = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00453}00453                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00454}00454                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00455}00455         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00456}00456         LOCK\_LOG\_TRY(\textcolor{stringliteral}{"{}XUPGRADE"{}}, \&sx-\/>lock\_object, 0, success, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00457}00457         \textcolor{keywordflow}{if} (success) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00458}00458                 curthread-\/>td\_sx\_slocks-\/-\/;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00459}00459                 WITNESS\_UPGRADE(\&sx-\/>lock\_object, LOP\_EXCLUSIVE | LOP\_TRYLOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00460}00460                     file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00461}00461                 LOCKSTAT\_RECORD0(sx\_\_upgrade, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00462}00462         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00463}00463         \textcolor{keywordflow}{return} (success);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00464}00464 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00465}00465 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00466}00466 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00467}\mbox{\hyperlink{kern__sx_8c_a4125d7b951c72f55b8714eec948b620b}{00467}} \mbox{\hyperlink{kern__sx_8c_a4125d7b951c72f55b8714eec948b620b}{sx\_try\_upgrade\_}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00468}00468 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00469}00469 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00470}00470         \textcolor{keywordflow}{return} (\mbox{\hyperlink{kern__sx_8c_a39f6a5d89e9fb03bc6cc567d185e7143}{sx\_try\_upgrade\_int}}(sx LOCK\_FILE\_LINE\_ARG));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00471}00471 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00472}00472 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00473}00473 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00474}00474 \textcolor{comment}{ * Downgrade an unrecursed exclusive lock into a single shared lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00475}00475 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00476}00476 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00477}\mbox{\hyperlink{kern__sx_8c_ad4c92a36ddf649258004410472f013e6}{00477}} \mbox{\hyperlink{kern__sx_8c_ad4c92a36ddf649258004410472f013e6}{sx\_downgrade\_int}}(\textcolor{keyword}{struct} sx *sx LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00478}00478 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00479}00479         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00480}00480         \textcolor{keywordtype}{int} wakeup\_swapper;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00481}00481 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00482}00482         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00483}00483                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00484}00484 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00485}00485         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00486}00486             (\textcolor{stringliteral}{"{}sx\_downgrade() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00487}00487         \mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{\_sx\_assert}}(sx, SA\_XLOCKED | SA\_NOTRECURSED, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00488}00488 \textcolor{preprocessor}{\#ifndef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00489}00489         \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a5745cf1e8f892977266d7fd5d04dbe2b}{sx\_recursed}}(sx))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00490}00490                 \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}downgrade of a recursed lock"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00491}00491 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00492}00492 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00493}00493         WITNESS\_DOWNGRADE(\&sx-\/>lock\_object, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00494}00494 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00495}00495         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00496}00496 \textcolor{comment}{         * Try to switch from an exclusive lock with no shared waiters}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00497}00497 \textcolor{comment}{         * to one sharer with no shared waiters.  If there are}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00498}00498 \textcolor{comment}{         * exclusive waiters, we don't need to lock the sleep queue so}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00499}00499 \textcolor{comment}{         * long as we preserve the flag.  We do one quick try and if}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00500}00500 \textcolor{comment}{         * that fails we grab the sleepq lock to keep the flags from}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00501}00501 \textcolor{comment}{         * changing and do it the slow way.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00502}00502 \textcolor{comment}{         *}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00503}00503 \textcolor{comment}{         * We have to lock the sleep queue if there are shared waiters}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00504}00504 \textcolor{comment}{         * so we can wake them up.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00505}00505 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00506}00506         x = sx-\/>sx\_lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00507}00507         \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED\_WAITERS) \&\&}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00508}00508             atomic\_cmpset\_rel\_ptr(\&sx-\/>sx\_lock, x, SX\_SHARERS\_LOCK(1) |}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00509}00509             (x \& SX\_LOCK\_EXCLUSIVE\_WAITERS)))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00510}00510                 \textcolor{keywordflow}{goto} out;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00511}00511 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00512}00512         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00513}00513 \textcolor{comment}{         * Lock the sleep queue so we can read the waiters bits}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00514}00514 \textcolor{comment}{         * without any races and wakeup any shared waiters.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00515}00515 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00516}00516         \mbox{\hyperlink{subr__sleepqueue_8c_ae0a612e63d885f5b440f1c0e84652baf}{sleepq\_lock}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00517}00517 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00518}00518         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00519}00519 \textcolor{comment}{         * Preserve SX\_LOCK\_EXCLUSIVE\_WAITERS while downgraded to a single}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00520}00520 \textcolor{comment}{         * shared lock.  If there are any shared waiters, wake them up.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00521}00521 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00522}00522         wakeup\_swapper = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00523}00523         x = sx-\/>sx\_lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00524}00524         atomic\_store\_rel\_ptr(\&sx-\/>sx\_lock, SX\_SHARERS\_LOCK(1) |}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00525}00525             (x \& SX\_LOCK\_EXCLUSIVE\_WAITERS));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00526}00526         \textcolor{keywordflow}{if} (x \& SX\_LOCK\_SHARED\_WAITERS)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00527}00527                 wakeup\_swapper = \mbox{\hyperlink{subr__sleepqueue_8c_a2c739ee96a9938ed1de78a8772311bd7}{sleepq\_broadcast}}(\&sx-\/>lock\_object, SLEEPQ\_SX,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00528}00528                     0, \mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{SQ\_SHARED\_QUEUE}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00529}00529         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00530}00530 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00531}00531         \textcolor{keywordflow}{if} (wakeup\_swapper)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00532}00532                 kick\_proc0();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00533}00533 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00534}00534 out:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00535}00535         curthread-\/>td\_sx\_slocks++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00536}00536         LOCK\_LOG\_LOCK(\textcolor{stringliteral}{"{}XDOWNGRADE"{}}, \&sx-\/>lock\_object, 0, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00537}00537         LOCKSTAT\_RECORD0(sx\_\_downgrade, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00538}00538 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00539}00539 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00540}00540 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00541}\mbox{\hyperlink{kern__sx_8c_a1f96db1ef32cb3f6db4f8117831dab33}{00541}} \mbox{\hyperlink{kern__sx_8c_a1f96db1ef32cb3f6db4f8117831dab33}{sx\_downgrade\_}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00542}00542 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00543}00543 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00544}00544         \mbox{\hyperlink{kern__sx_8c_ad4c92a36ddf649258004410472f013e6}{sx\_downgrade\_int}}(sx LOCK\_FILE\_LINE\_ARG);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00545}00545 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00546}00546 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00547}00547 \textcolor{preprocessor}{\#ifdef  ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00548}00548 \textcolor{keyword}{static} \textcolor{keyword}{inline} \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00549}00549 \mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{sx\_drop\_critical}}(uintptr\_t x, \textcolor{keywordtype}{bool} *in\_critical, \textcolor{keywordtype}{int} *extra\_work)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00550}00550 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00551}00551 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00552}00552         \textcolor{keywordflow}{if} (x \& SX\_LOCK\_WRITE\_SPINNER)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00553}00553                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00554}00554         \textcolor{keywordflow}{if} (*in\_critical) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00555}00555                 critical\_exit();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00556}00556                 *in\_critical = \textcolor{keyword}{false};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00557}00557                 (*extra\_work)-\/-\/;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00558}00558         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00559}00559 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00560}00560 \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00561}\mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{00561}} \textcolor{preprocessor}{\#define sx\_drop\_critical(x, in\_critical, extra\_work) do \{ \} while (0)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00562}00562 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00563}00563 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00564}00564 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00565}00565 \textcolor{comment}{ * This function represents the so-\/called 'hard case' for sx\_xlock}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00566}00566 \textcolor{comment}{ * operation.  All 'easy case' failures are redirected to this.  Note}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00567}00567 \textcolor{comment}{ * that ideally this would be a static function, but it needs to be}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00568}00568 \textcolor{comment}{ * accessible from at least sx.h.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00569}00569 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00570}00570 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00571}\mbox{\hyperlink{kern__sx_8c_acc52958e610760d9e978ecdb14365bc9}{00571}} \mbox{\hyperlink{kern__sx_8c_acc52958e610760d9e978ecdb14365bc9}{\_sx\_xlock\_hard}}(\textcolor{keyword}{struct} sx *sx, uintptr\_t x, \textcolor{keywordtype}{int} opts LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00572}00572 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00573}00573         \mbox{\hyperlink{kern__sx_8c_a9cc3c95f2349916c70a759a449314da5}{GIANT\_DECLARE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00574}00574         uintptr\_t tid, setx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00575}00575 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00576}00576         \textcolor{keyword}{struct }thread *owner;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00577}00577         u\_int i, n, spintries = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00578}00578         \textcolor{keyword}{enum} \{ READERS, WRITER \} sleep\_reason = READERS;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00579}00579         \textcolor{keywordtype}{bool} in\_critical = \textcolor{keyword}{false};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00580}00580 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00581}00581 \textcolor{preprocessor}{\#ifdef LOCK\_PROFILING}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00582}00582         uint64\_t waittime = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00583}00583         \textcolor{keywordtype}{int} contested = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00584}00584 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00585}00585         \textcolor{keywordtype}{int} error = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00586}00586 \textcolor{preprocessor}{\#if defined(ADAPTIVE\_SX) || defined(KDTRACE\_HOOKS)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00587}00587         \textcolor{keyword}{struct }lock\_delay\_arg lda;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00588}00588 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00589}00589 \textcolor{preprocessor}{\#ifdef  KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00590}00590         u\_int sleep\_cnt = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00591}00591         int64\_t sleep\_time = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00592}00592         int64\_t all\_time = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00593}00593 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00594}00594 \textcolor{preprocessor}{\#if defined(KDTRACE\_HOOKS) || defined(LOCK\_PROFILING)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00595}00595         uintptr\_t state = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00596}00596         \textcolor{keywordtype}{int} doing\_lockprof = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00597}00597 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00598}00598         \textcolor{keywordtype}{int} extra\_work = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00599}00599 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00600}00600         tid = (uintptr\_t)curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00601}00601 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00602}00602 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00603}00603         \textcolor{keywordflow}{if} (LOCKSTAT\_PROFILE\_ENABLED(sx\_\_acquire)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00604}00604                 \textcolor{keywordflow}{while} (x == SX\_LOCK\_UNLOCKED) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00605}00605                         \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00606}00606                                 \textcolor{keywordflow}{goto} out\_lockstat;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00607}00607                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00608}00608                 extra\_work = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00609}00609                 doing\_lockprof = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00610}00610                 all\_time -\/= \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00611}00611                 state = x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00612}00612         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00613}00613 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00614}00614 \textcolor{preprocessor}{\#ifdef LOCK\_PROFILING}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00615}00615         extra\_work = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00616}00616         doing\_lockprof = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00617}00617         state = x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00618}00618 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00619}00619 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00620}00620         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00621}00621                 \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00622}00622 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00623}00623         \textcolor{keywordflow}{if} (\_\_predict\_false(x == SX\_LOCK\_UNLOCKED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00624}00624                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00625}00625 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00626}00626         \textcolor{comment}{/* If we already hold an exclusive lock, then recurse. */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00627}00627         \textcolor{keywordflow}{if} (\_\_predict\_false(lv\_sx\_owner(x) == (\textcolor{keyword}{struct} thread *)tid)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00628}00628                 KASSERT((sx-\/>lock\_object.lo\_flags \& LO\_RECURSABLE) != 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00629}00629             (\textcolor{stringliteral}{"{}\_sx\_xlock\_hard: recursed on non-\/recursive sx \%s @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00630}00630                     sx-\/>lock\_object.lo\_name, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00631}00631                 sx-\/>sx\_recurse++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00632}00632                 atomic\_set\_ptr(\&sx-\/>sx\_lock, SX\_LOCK\_RECURSED);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00633}00633                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00634}00634                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p recursing"{}}, \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00635}00635                 \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00636}00636         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00637}00637 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00638}00638         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00639}00639                 CTR5(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%s contested (lock=\%p) at \%s:\%d"{}}, \_\_func\_\_,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00640}00640                     sx-\/>lock\_object.lo\_name, (\textcolor{keywordtype}{void} *)sx-\/>sx\_lock, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00641}00641 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00642}00642 \textcolor{preprocessor}{\#if defined(ADAPTIVE\_SX)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00643}00643         lock\_delay\_arg\_init(\&lda, \&sx\_delay);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00644}00644 \textcolor{preprocessor}{\#elif defined(KDTRACE\_HOOKS)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00645}00645         lock\_delay\_arg\_init\_noadapt(\&lda);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00646}00646 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00647}00647 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00648}00648 \textcolor{preprocessor}{\#ifdef HWPMC\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00649}00649         PMC\_SOFT\_CALL( , , lock, failed);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00650}00650 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00651}00651         lock\_profile\_obtain\_lock\_failed(\&sx-\/>lock\_object, \textcolor{keyword}{false}, \&contested,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00652}00652             \&waittime);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00653}00653 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00654}00654 \textcolor{preprocessor}{\#ifndef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00655}00655         \mbox{\hyperlink{kern__sx_8c_a0f4702099e1ba4bb91339c5e1139caa0}{GIANT\_SAVE}}(extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00656}00656 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00657}00657 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00658}00658         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00659}00659                 \textcolor{keywordflow}{if} (x == SX\_LOCK\_UNLOCKED) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00660}00660                         \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00661}00661                                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00662}00662                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00663}00663                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00664}00664 \textcolor{preprocessor}{\#ifdef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00665}00665                 \mbox{\hyperlink{kern__sx_8c_a0f4702099e1ba4bb91339c5e1139caa0}{GIANT\_SAVE}}(extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00666}00666 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00667}00667 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00668}00668                 lda.spin\_cnt++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00669}00669 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00670}00670 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00671}00671                 \textcolor{keywordflow}{if} (x == (SX\_LOCK\_SHARED | SX\_LOCK\_WRITE\_SPINNER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00672}00672                         \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00673}00673                                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00674}00674                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00675}00675                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00676}00676 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00677}00677                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00678}00678 \textcolor{comment}{                 * If the lock is write locked and the owner is}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00679}00679 \textcolor{comment}{                 * running on another CPU, spin until the owner stops}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00680}00680 \textcolor{comment}{                 * running or the state of the lock changes.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00681}00681 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00682}00682                 \textcolor{keywordflow}{if} ((x \& SX\_LOCK\_SHARED) == 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00683}00683                         \mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{sx\_drop\_critical}}(x, \&in\_critical, \&extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00684}00684                         sleep\_reason = WRITER;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00685}00685                         owner = lv\_sx\_owner(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00686}00686                         \textcolor{keywordflow}{if} (!TD\_IS\_RUNNING(owner))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00687}00687                                 \textcolor{keywordflow}{goto} sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00688}00688                         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00689}00689                                 CTR3(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: spinning on \%p held by \%p"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00690}00690                                     \_\_func\_\_, sx, owner);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00691}00691                         KTR\_STATE1(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00692}00692                             \textcolor{stringliteral}{"{}spinning"{}}, \textcolor{stringliteral}{"{}lockname:\(\backslash\)"{}\%s\(\backslash\)"{}"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00693}00693                             sx-\/>lock\_object.lo\_name);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00694}00694                         \textcolor{keywordflow}{do} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00695}00695                                 \mbox{\hyperlink{subr__lock_8c_aafae06c4e465128885fccaf5f9e482bb}{lock\_delay}}(\&lda);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00696}00696                                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00697}00697                                 owner = lv\_sx\_owner(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00698}00698                         \} \textcolor{keywordflow}{while} (owner != NULL \&\& TD\_IS\_RUNNING(owner));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00699}00699                         KTR\_STATE0(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00700}00700                             \textcolor{stringliteral}{"{}running"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00701}00701                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00702}00702                 \} \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (SX\_SHARERS(x) > 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00703}00703                         sleep\_reason = READERS;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00704}00704                         \textcolor{keywordflow}{if} (spintries == asx\_retries)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00705}00705                                 \textcolor{keywordflow}{goto} sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00706}00706                         \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_WRITE\_SPINNER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00707}00707                                 \textcolor{keywordflow}{if} (!in\_critical) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00708}00708                                         critical\_enter();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00709}00709                                         in\_critical = \textcolor{keyword}{true};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00710}00710                                         extra\_work++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00711}00711                                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00712}00712                                 \textcolor{keywordflow}{if} (!atomic\_fcmpset\_ptr(\&sx-\/>sx\_lock, \&x,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00713}00713                                     x | SX\_LOCK\_WRITE\_SPINNER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00714}00714                                         critical\_exit();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00715}00715                                         in\_critical = \textcolor{keyword}{false};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00716}00716                                         extra\_work-\/-\/;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00717}00717                                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00718}00718                                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00719}00719                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00720}00720                         spintries++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00721}00721                         KTR\_STATE1(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00722}00722                             \textcolor{stringliteral}{"{}spinning"{}}, \textcolor{stringliteral}{"{}lockname:\(\backslash\)"{}\%s\(\backslash\)"{}"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00723}00723                             sx-\/>lock\_object.lo\_name);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00724}00724                         n = SX\_SHARERS(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00725}00725                         \textcolor{keywordflow}{for} (i = 0; i < asx\_loops; i += n) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00726}00726                                 lock\_delay\_spin(n);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00727}00727                                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00728}00728                                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_WRITE\_SPINNER))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00729}00729                                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00730}00730                                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00731}00731                                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00732}00732                                 n = SX\_SHARERS(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00733}00733                                 \textcolor{keywordflow}{if} (n == 0)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00734}00734                                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00735}00735                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00736}00736 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00737}00737                         lda.spin\_cnt += i;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00738}00738 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00739}00739                         KTR\_STATE0(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00740}00740                             \textcolor{stringliteral}{"{}running"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00741}00741                         \textcolor{keywordflow}{if} (i < asx\_loops)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00742}00742                                 \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00743}00743                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00744}00744 sleepq:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00745}00745 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00746}00746                 \mbox{\hyperlink{subr__sleepqueue_8c_ae0a612e63d885f5b440f1c0e84652baf}{sleepq\_lock}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00747}00747                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00748}00748 retry\_sleepq:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00749}00749 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00750}00750                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00751}00751 \textcolor{comment}{                 * If the lock was released while spinning on the}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00752}00752 \textcolor{comment}{                 * sleep queue chain lock, try again.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00753}00753 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00754}00754                 \textcolor{keywordflow}{if} (x == SX\_LOCK\_UNLOCKED) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00755}00755                         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00756}00756                         \mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{sx\_drop\_critical}}(x, \&in\_critical, \&extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00757}00757                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00758}00758                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00759}00759 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00760}00760 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00761}00761                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00762}00762 \textcolor{comment}{                 * The current lock owner might have started executing}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00763}00763 \textcolor{comment}{                 * on another CPU (or the lock could have changed}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00764}00764 \textcolor{comment}{                 * owners) while we were waiting on the sleep queue}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00765}00765 \textcolor{comment}{                 * chain lock.  If so, drop the sleep queue lock and try}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00766}00766 \textcolor{comment}{                 * again.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00767}00767 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00768}00768                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00769}00769                         owner = (\textcolor{keyword}{struct }thread *)SX\_OWNER(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00770}00770                         \textcolor{keywordflow}{if} (TD\_IS\_RUNNING(owner)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00771}00771                                 \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00772}00772                                 \mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{sx\_drop\_critical}}(x, \&in\_critical,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00773}00773                                     \&extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00774}00774                                 \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00775}00775                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00776}00776                 \} \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (SX\_SHARERS(x) > 0 \&\& sleep\_reason == WRITER) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00777}00777                         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00778}00778                         \mbox{\hyperlink{kern__sx_8c_a19a005c9232cefb0c074c02942bdfd16}{sx\_drop\_critical}}(x, \&in\_critical, \&extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00779}00779                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00780}00780                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00781}00781 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00782}00782 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00783}00783                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00784}00784 \textcolor{comment}{                 * If an exclusive lock was released with both shared}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00785}00785 \textcolor{comment}{                 * and exclusive waiters and a shared waiter hasn't}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00786}00786 \textcolor{comment}{                 * woken up and acquired the lock yet, sx\_lock will be}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00787}00787 \textcolor{comment}{                 * set to SX\_LOCK\_UNLOCKED | SX\_LOCK\_EXCLUSIVE\_WAITERS.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00788}00788 \textcolor{comment}{                 * If we see that value, try to acquire it once.  Note}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00789}00789 \textcolor{comment}{                 * that we have to preserve SX\_LOCK\_EXCLUSIVE\_WAITERS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00790}00790 \textcolor{comment}{                 * as there are other exclusive waiters still.  If we}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00791}00791 \textcolor{comment}{                 * fail, restart the loop.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00792}00792 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00793}00793                 setx = x \& (SX\_LOCK\_WAITERS | SX\_LOCK\_WRITE\_SPINNER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00794}00794                 \textcolor{keywordflow}{if} ((x \& \string~setx) == SX\_LOCK\_SHARED) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00795}00795                         setx \&= \string~SX\_LOCK\_WRITE\_SPINNER;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00796}00796                         \textcolor{keywordflow}{if} (!atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, \&x, tid | setx))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00797}00797                                 \textcolor{keywordflow}{goto} retry\_sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00798}00798                         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00799}00799                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p claimed by new writer"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00800}00800                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00801}00801                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00802}00802                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00803}00803 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00804}00804 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00805}00805                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00806}00806 \textcolor{comment}{                 * It is possible we set the SX\_LOCK\_WRITE\_SPINNER bit.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00807}00807 \textcolor{comment}{                 * It is an invariant that when the bit is set, there is}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00808}00808 \textcolor{comment}{                 * a writer ready to grab the lock. Thus clear the bit since}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00809}00809 \textcolor{comment}{                 * we are going to sleep.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00810}00810 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00811}00811                 \textcolor{keywordflow}{if} (in\_critical) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00812}00812                         \textcolor{keywordflow}{if} ((x \& SX\_LOCK\_WRITE\_SPINNER) ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00813}00813                             !((x \& SX\_LOCK\_EXCLUSIVE\_WAITERS))) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00814}00814                                 setx = x \& \string~SX\_LOCK\_WRITE\_SPINNER;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00815}00815                                 setx |= SX\_LOCK\_EXCLUSIVE\_WAITERS;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00816}00816                                 \textcolor{keywordflow}{if} (!atomic\_fcmpset\_ptr(\&sx-\/>sx\_lock, \&x,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00817}00817                                     setx)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00818}00818                                         \textcolor{keywordflow}{goto} retry\_sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00819}00819                                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00820}00820                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00821}00821                         critical\_exit();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00822}00822                         in\_critical = \textcolor{keyword}{false};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00823}00823                 \} \textcolor{keywordflow}{else} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00824}00824 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00825}00825                         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00826}00826 \textcolor{comment}{                         * Try to set the SX\_LOCK\_EXCLUSIVE\_WAITERS.  If we fail,}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00827}00827 \textcolor{comment}{                         * than loop back and retry.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00828}00828 \textcolor{comment}{                         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00829}00829                         \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_EXCLUSIVE\_WAITERS)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00830}00830                                 \textcolor{keywordflow}{if} (!atomic\_fcmpset\_ptr(\&sx-\/>sx\_lock, \&x,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00831}00831                                     x | SX\_LOCK\_EXCLUSIVE\_WAITERS)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00832}00832                                         \textcolor{keywordflow}{goto} retry\_sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00833}00833                                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00834}00834                                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00835}00835                                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p set excl waiters flag"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00836}00836                                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00837}00837                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00838}00838 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00839}00839                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00840}00840 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00841}00841 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00842}00842                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00843}00843 \textcolor{comment}{                 * Since we have been unable to acquire the exclusive}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00844}00844 \textcolor{comment}{                 * lock and the exclusive waiters flag is set, we have}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00845}00845 \textcolor{comment}{                 * to sleep.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00846}00846 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00847}00847                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00848}00848                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p blocking on sleep queue"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00849}00849                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00850}00850 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00851}00851 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00852}00852                 sleep\_time -\/= \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00853}00853 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00854}00854                 \mbox{\hyperlink{subr__sleepqueue_8c_aa2822b5d83f7fb459964aa82b02b8701}{sleepq\_add}}(\&sx-\/>lock\_object, NULL, sx-\/>lock\_object.lo\_name,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00855}00855                     SLEEPQ\_SX | ((opts \& SX\_INTERRUPTIBLE) ?}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00856}00856                     SLEEPQ\_INTERRUPTIBLE : 0), \mbox{\hyperlink{kern__sx_8c_a3cde1e96adc94f05ec2d174559d9cb8d}{SQ\_EXCLUSIVE\_QUEUE}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00857}00857                 \textcolor{keywordflow}{if} (!(opts \& SX\_INTERRUPTIBLE))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00858}00858                         \mbox{\hyperlink{subr__sleepqueue_8c_ac839176dcd4b987ba04b3112b5502303}{sleepq\_wait}}(\&sx-\/>lock\_object, 0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00859}00859                 \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00860}00860                         error = \mbox{\hyperlink{subr__sleepqueue_8c_a1ceaeff8d86d1d875208f14e9342ded6}{sleepq\_wait\_sig}}(\&sx-\/>lock\_object, 0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00861}00861 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00862}00862                 sleep\_time += \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00863}00863                 sleep\_cnt++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00864}00864 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00865}00865                 \textcolor{keywordflow}{if} (error) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00866}00866                         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00867}00867                                 CTR2(KTR\_LOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00868}00868                         \textcolor{stringliteral}{"{}\%s: interruptible sleep by \%p suspended by signal"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00869}00869                                     \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00870}00870                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00871}00871                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00872}00872                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00873}00873                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p resuming from sleep queue"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00874}00874                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00875}00875                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00876}00876         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00877}00877         \textcolor{keywordflow}{if} (\_\_predict\_true(!extra\_work))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00878}00878                 \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00879}00879 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00880}00880         \textcolor{keywordflow}{if} (in\_critical)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00881}00881                 critical\_exit();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00882}00882 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00883}00883         \mbox{\hyperlink{kern__sx_8c_a4071a8f3605a24fcdac5e4cba9d0526f}{GIANT\_RESTORE}}();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00884}00884 \textcolor{preprocessor}{\#if defined(KDTRACE\_HOOKS) || defined(LOCK\_PROFILING)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00885}00885         \textcolor{keywordflow}{if} (\_\_predict\_true(!doing\_lockprof))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00886}00886                 \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00887}00887 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00888}00888 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00889}00889         all\_time += \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00890}00890         \textcolor{keywordflow}{if} (sleep\_time)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00891}00891                 LOCKSTAT\_RECORD4(sx\_\_block, sx, sleep\_time,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00892}00892                     LOCKSTAT\_WRITER, (state \& SX\_LOCK\_SHARED) == 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00893}00893                     (state \& SX\_LOCK\_SHARED) == 0 ? 0 : SX\_SHARERS(state));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00894}00894         \textcolor{keywordflow}{if} (lda.spin\_cnt > sleep\_cnt)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00895}00895                 LOCKSTAT\_RECORD4(sx\_\_spin, sx, all\_time -\/ sleep\_time,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00896}00896                     LOCKSTAT\_WRITER, (state \& SX\_LOCK\_SHARED) == 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00897}00897                     (state \& SX\_LOCK\_SHARED) == 0 ? 0 : SX\_SHARERS(state));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00898}00898 out\_lockstat:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00899}00899 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00900}00900         \textcolor{keywordflow}{if} (!error)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00901}00901                 LOCKSTAT\_PROFILE\_OBTAIN\_RWLOCK\_SUCCESS(sx\_\_acquire, sx,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00902}00902                     contested, waittime, file, line, LOCKSTAT\_WRITER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00903}00903         \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00904}00904 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00905}00905 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00906}00906 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00907}00907 \textcolor{comment}{ * This function represents the so-\/called 'hard case' for sx\_xunlock}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00908}00908 \textcolor{comment}{ * operation.  All 'easy case' failures are redirected to this.  Note}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00909}00909 \textcolor{comment}{ * that ideally this would be a static function, but it needs to be}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00910}00910 \textcolor{comment}{ * accessible from at least sx.h.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00911}00911 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00912}00912 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00913}\mbox{\hyperlink{kern__sx_8c_a038866f68f0a49227c3ec51014e077d3}{00913}} \mbox{\hyperlink{kern__sx_8c_a038866f68f0a49227c3ec51014e077d3}{\_sx\_xunlock\_hard}}(\textcolor{keyword}{struct} sx *sx, uintptr\_t x LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00914}00914 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00915}00915         uintptr\_t tid, setx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00916}00916         \textcolor{keywordtype}{int} queue, wakeup\_swapper;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00917}00917 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00918}00918         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00919}00919                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00920}00920 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00921}00921         tid = (uintptr\_t)curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00922}00922 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00923}00923         \textcolor{keywordflow}{if} (\_\_predict\_false(x == tid))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00924}00924                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00925}00925 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00926}00926         MPASS(!(x \& SX\_LOCK\_SHARED));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00927}00927 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00928}00928         \textcolor{keywordflow}{if} (\_\_predict\_false(x \& SX\_LOCK\_RECURSED)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00929}00929                 \textcolor{comment}{/* The lock is recursed, unrecurse one level. */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00930}00930                 \textcolor{keywordflow}{if} ((-\/-\/sx-\/>sx\_recurse) == 0)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00931}00931                         atomic\_clear\_ptr(\&sx-\/>sx\_lock, SX\_LOCK\_RECURSED);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00932}00932                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00933}00933                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p unrecursing"{}}, \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00934}00934                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00935}00935         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00936}00936 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00937}00937         LOCKSTAT\_PROFILE\_RELEASE\_RWLOCK(sx\_\_release, sx, LOCKSTAT\_WRITER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00938}00938         \textcolor{keywordflow}{if} (x == tid \&\&}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00939}00939             atomic\_cmpset\_rel\_ptr(\&sx-\/>sx\_lock, tid, SX\_LOCK\_UNLOCKED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00940}00940                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00941}00941 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00942}00942         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00943}00943                 CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p contested"{}}, \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00944}00944 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00945}00945         \mbox{\hyperlink{subr__sleepqueue_8c_ae0a612e63d885f5b440f1c0e84652baf}{sleepq\_lock}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00946}00946         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00947}00947         MPASS(x \& (SX\_LOCK\_SHARED\_WAITERS | SX\_LOCK\_EXCLUSIVE\_WAITERS));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00948}00948 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00949}00949         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00950}00950 \textcolor{comment}{         * The wake up algorithm here is quite simple and probably not}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00951}00951 \textcolor{comment}{         * ideal.  It gives precedence to shared waiters if they are}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00952}00952 \textcolor{comment}{         * present.  For this condition, we have to preserve the}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00953}00953 \textcolor{comment}{         * state of the exclusive waiters flag.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00954}00954 \textcolor{comment}{         * If interruptible sleeps left the shared queue empty avoid a}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00955}00955 \textcolor{comment}{         * starvation for the threads sleeping on the exclusive queue by giving}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00956}00956 \textcolor{comment}{         * them precedence and cleaning up the shared waiters bit anyway.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00957}00957 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00958}00958         setx = SX\_LOCK\_UNLOCKED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00959}00959         queue = \mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{SQ\_SHARED\_QUEUE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00960}00960         \textcolor{keywordflow}{if} ((x \& SX\_LOCK\_EXCLUSIVE\_WAITERS) != 0 \&\&}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00961}00961             \mbox{\hyperlink{subr__sleepqueue_8c_a9705dc06b2f7af3c15a9239c7fdae058}{sleepq\_sleepcnt}}(\&sx-\/>lock\_object, \mbox{\hyperlink{kern__sx_8c_a3cde1e96adc94f05ec2d174559d9cb8d}{SQ\_EXCLUSIVE\_QUEUE}}) != 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00962}00962                 queue = \mbox{\hyperlink{kern__sx_8c_a3cde1e96adc94f05ec2d174559d9cb8d}{SQ\_EXCLUSIVE\_QUEUE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00963}00963                 setx |= (x \& SX\_LOCK\_SHARED\_WAITERS);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00964}00964         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00965}00965         atomic\_store\_rel\_ptr(\&sx-\/>sx\_lock, setx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00966}00966 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00967}00967         \textcolor{comment}{/* Wake up all the waiters for the specific queue. */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00968}00968         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00969}00969                 CTR3(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p waking up all threads on \%s queue"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00970}00970                     \_\_func\_\_, sx, queue == \mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{SQ\_SHARED\_QUEUE}} ? \textcolor{stringliteral}{"{}shared"{}} :}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00971}00971                     \textcolor{stringliteral}{"{}exclusive"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00972}00972 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00973}00973         wakeup\_swapper = \mbox{\hyperlink{subr__sleepqueue_8c_a2c739ee96a9938ed1de78a8772311bd7}{sleepq\_broadcast}}(\&sx-\/>lock\_object, SLEEPQ\_SX, 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00974}00974             queue);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00975}00975         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00976}00976         \textcolor{keywordflow}{if} (wakeup\_swapper)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00977}00977                 kick\_proc0();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00978}00978 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00979}00979 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00980}00980 \textcolor{keyword}{static} \textcolor{keywordtype}{bool} \_\_always\_inline}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00981}\mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{00981}} \mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{\_\_sx\_can\_read}}(\textcolor{keyword}{struct} thread *td, uintptr\_t x, \textcolor{keywordtype}{bool} fp)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00982}00982 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00983}00983 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00984}00984         \textcolor{keywordflow}{if} ((x \& (SX\_LOCK\_SHARED | SX\_LOCK\_EXCLUSIVE\_WAITERS | SX\_LOCK\_WRITE\_SPINNER))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00985}00985                         == SX\_LOCK\_SHARED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00986}00986                 \textcolor{keywordflow}{return} (\textcolor{keyword}{true});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00987}00987         \textcolor{keywordflow}{if} (!fp \&\& td-\/>td\_sx\_slocks \&\& (x \& SX\_LOCK\_SHARED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00988}00988                 \textcolor{keywordflow}{return} (\textcolor{keyword}{true});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00989}00989         \textcolor{keywordflow}{return} (\textcolor{keyword}{false});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00990}00990 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00991}00991 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00992}00992 \textcolor{keyword}{static} \textcolor{keywordtype}{bool} \_\_always\_inline}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00993}\mbox{\hyperlink{kern__sx_8c_a6ed0837b4bb3b770089354c33a1b3063}{00993}} \mbox{\hyperlink{kern__sx_8c_a6ed0837b4bb3b770089354c33a1b3063}{\_\_sx\_slock\_try}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{struct} thread *td, uintptr\_t *xp, \textcolor{keywordtype}{bool} fp}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00994}00994     LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00995}00995 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00996}00996 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00997}00997         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00998}00998 \textcolor{comment}{         * If no other thread has an exclusive lock then try to bump up}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l00999}00999 \textcolor{comment}{         * the count of sharers.  Since we have to preserve the state}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01000}01000 \textcolor{comment}{         * of SX\_LOCK\_EXCLUSIVE\_WAITERS, if we fail to acquire the}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01001}01001 \textcolor{comment}{         * shared lock loop back and retry.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01002}01002 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01003}01003         \textcolor{keywordflow}{while} (\mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{\_\_sx\_can\_read}}(td, *xp, fp)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01004}01004                 \textcolor{keywordflow}{if} (atomic\_fcmpset\_acq\_ptr(\&sx-\/>sx\_lock, xp,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01005}01005                     *xp + SX\_ONE\_SHARER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01006}01006                         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01007}01007                                 CTR4(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p succeed \%p -\/> \%p"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01008}01008                                     \_\_func\_\_, sx, (\textcolor{keywordtype}{void} *)*xp,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01009}01009                                     (\textcolor{keywordtype}{void} *)(*xp + SX\_ONE\_SHARER));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01010}01010                         td-\/>td\_sx\_slocks++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01011}01011                         \textcolor{keywordflow}{return} (\textcolor{keyword}{true});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01012}01012                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01013}01013         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01014}01014         \textcolor{keywordflow}{return} (\textcolor{keyword}{false});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01015}01015 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01016}01016 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01017}01017 \textcolor{keyword}{static} \textcolor{keywordtype}{int} \_\_noinline}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01018}\mbox{\hyperlink{kern__sx_8c_a112785d75887c2f9917ba399e1a29c3f}{01018}} \mbox{\hyperlink{kern__sx_8c_a112785d75887c2f9917ba399e1a29c3f}{\_sx\_slock\_hard}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keywordtype}{int} opts, uintptr\_t x LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01019}01019 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01020}01020         \mbox{\hyperlink{kern__sx_8c_a9cc3c95f2349916c70a759a449314da5}{GIANT\_DECLARE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01021}01021         \textcolor{keyword}{struct }thread *td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01022}01022 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01023}01023         \textcolor{keyword}{struct }thread *owner;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01024}01024         u\_int i, n, spintries = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01025}01025 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01026}01026 \textcolor{preprocessor}{\#ifdef LOCK\_PROFILING}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01027}01027         uint64\_t waittime = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01028}01028         \textcolor{keywordtype}{int} contested = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01029}01029 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01030}01030         \textcolor{keywordtype}{int} error = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01031}01031 \textcolor{preprocessor}{\#if defined(ADAPTIVE\_SX) || defined(KDTRACE\_HOOKS)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01032}01032         \textcolor{keyword}{struct }lock\_delay\_arg lda;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01033}01033 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01034}01034 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01035}01035         u\_int sleep\_cnt = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01036}01036         int64\_t sleep\_time = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01037}01037         int64\_t all\_time = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01038}01038 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01039}01039 \textcolor{preprocessor}{\#if defined(KDTRACE\_HOOKS) || defined(LOCK\_PROFILING)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01040}01040         uintptr\_t state = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01041}01041 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01042}01042         \textcolor{keywordtype}{int} extra\_work = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01043}01043 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01044}01044         td = curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01045}01045 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01046}01046 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01047}01047         \textcolor{keywordflow}{if} (LOCKSTAT\_PROFILE\_ENABLED(sx\_\_acquire)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01048}01048                 \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a6ed0837b4bb3b770089354c33a1b3063}{\_\_sx\_slock\_try}}(sx, td, \&x, \textcolor{keyword}{false} LOCK\_FILE\_LINE\_ARG))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01049}01049                         \textcolor{keywordflow}{goto} out\_lockstat;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01050}01050                 extra\_work = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01051}01051                 all\_time -\/= \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01052}01052                 state = x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01053}01053         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01054}01054 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01055}01055 \textcolor{preprocessor}{\#ifdef LOCK\_PROFILING}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01056}01056         extra\_work = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01057}01057         state = x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01058}01058 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01059}01059 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01060}01060         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01061}01061                 \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01062}01062 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01063}01063 \textcolor{preprocessor}{\#if defined(ADAPTIVE\_SX)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01064}01064         lock\_delay\_arg\_init(\&lda, \&sx\_delay);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01065}01065 \textcolor{preprocessor}{\#elif defined(KDTRACE\_HOOKS)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01066}01066         lock\_delay\_arg\_init\_noadapt(\&lda);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01067}01067 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01068}01068 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01069}01069 \textcolor{preprocessor}{\#ifdef HWPMC\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01070}01070         PMC\_SOFT\_CALL( , , lock, failed);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01071}01071 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01072}01072         lock\_profile\_obtain\_lock\_failed(\&sx-\/>lock\_object, \textcolor{keyword}{false}, \&contested,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01073}01073             \&waittime);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01074}01074 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01075}01075 \textcolor{preprocessor}{\#ifndef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01076}01076         \mbox{\hyperlink{kern__sx_8c_a0f4702099e1ba4bb91339c5e1139caa0}{GIANT\_SAVE}}(extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01077}01077 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01078}01078 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01079}01079         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01080}01080 \textcolor{comment}{         * As with rwlocks, we don't make any attempt to try to block}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01081}01081 \textcolor{comment}{         * shared locks once there is an exclusive waiter.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01082}01082 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01083}01083         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01084}01084                 \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a6ed0837b4bb3b770089354c33a1b3063}{\_\_sx\_slock\_try}}(sx, td, \&x, \textcolor{keyword}{false} LOCK\_FILE\_LINE\_ARG))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01085}01085                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01086}01086 \textcolor{preprocessor}{\#ifdef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01087}01087                 \mbox{\hyperlink{kern__sx_8c_a0f4702099e1ba4bb91339c5e1139caa0}{GIANT\_SAVE}}(extra\_work);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01088}01088 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01089}01089 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01090}01090                 lda.spin\_cnt++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01091}01091 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01092}01092 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01093}01093 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01094}01094                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01095}01095 \textcolor{comment}{                 * If the owner is running on another CPU, spin until}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01096}01096 \textcolor{comment}{                 * the owner stops running or the state of the lock}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01097}01097 \textcolor{comment}{                 * changes.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01098}01098 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01099}01099                 \textcolor{keywordflow}{if} ((x \& SX\_LOCK\_SHARED) == 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01100}01100                         owner = lv\_sx\_owner(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01101}01101                         \textcolor{keywordflow}{if} (TD\_IS\_RUNNING(owner)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01102}01102                                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01103}01103                                         CTR3(KTR\_LOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01104}01104                                             \textcolor{stringliteral}{"{}\%s: spinning on \%p held by \%p"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01105}01105                                             \_\_func\_\_, sx, owner);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01106}01106                                 KTR\_STATE1(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01107}01107                                     \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread), \textcolor{stringliteral}{"{}spinning"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01108}01108                                     \textcolor{stringliteral}{"{}lockname:\(\backslash\)"{}\%s\(\backslash\)"{}"{}}, sx-\/>lock\_object.lo\_name);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01109}01109                                 \textcolor{keywordflow}{do} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01110}01110                                         \mbox{\hyperlink{subr__lock_8c_aafae06c4e465128885fccaf5f9e482bb}{lock\_delay}}(\&lda);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01111}01111                                         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01112}01112                                         owner = lv\_sx\_owner(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01113}01113                                 \} \textcolor{keywordflow}{while} (owner != NULL \&\& TD\_IS\_RUNNING(owner));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01114}01114                                 KTR\_STATE0(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01115}01115                                     \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread), \textcolor{stringliteral}{"{}running"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01116}01116                                 \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01117}01117                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01118}01118                 \} \textcolor{keywordflow}{else} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01119}01119                         \textcolor{keywordflow}{if} ((x \& SX\_LOCK\_WRITE\_SPINNER) \&\& SX\_SHARERS(x) == 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01120}01120                                 MPASS(!\mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{\_\_sx\_can\_read}}(td, x, \textcolor{keyword}{false}));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01121}01121                                 lock\_delay\_spin(2);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01122}01122                                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01123}01123                                 \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01124}01124                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01125}01125                         \textcolor{keywordflow}{if} (spintries < asx\_retries) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01126}01126                                 KTR\_STATE1(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01127}01127                                     \textcolor{stringliteral}{"{}spinning"{}}, \textcolor{stringliteral}{"{}lockname:\(\backslash\)"{}\%s\(\backslash\)"{}"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01128}01128                                     sx-\/>lock\_object.lo\_name);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01129}01129                                 n = SX\_SHARERS(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01130}01130                                 \textcolor{keywordflow}{for} (i = 0; i < asx\_loops; i += n) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01131}01131                                         lock\_delay\_spin(n);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01132}01132                                         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01133}01133                                         \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01134}01134                                                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01135}01135                                         n = SX\_SHARERS(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01136}01136                                         \textcolor{keywordflow}{if} (n == 0)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01137}01137                                                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01138}01138                                         \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{\_\_sx\_can\_read}}(td, x, \textcolor{keyword}{false}))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01139}01139                                                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01140}01140                                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01141}01141 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01142}01142                                 lda.spin\_cnt += i;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01143}01143 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01144}01144                                 KTR\_STATE0(KTR\_SCHED, \textcolor{stringliteral}{"{}thread"{}}, \mbox{\hyperlink{sched__4bsd_8c_a9be90c0f78ead61b5d64f6e7c21996fe}{sched\_tdname}}(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01145}01145                                     \textcolor{stringliteral}{"{}running"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01146}01146                                 \textcolor{keywordflow}{if} (i < asx\_loops)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01147}01147                                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01148}01148                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01149}01149                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01150}01150 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01151}01151 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01152}01152                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01153}01153 \textcolor{comment}{                 * Some other thread already has an exclusive lock, so}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01154}01154 \textcolor{comment}{                 * start the process of blocking.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01155}01155 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01156}01156                 \mbox{\hyperlink{subr__sleepqueue_8c_ae0a612e63d885f5b440f1c0e84652baf}{sleepq\_lock}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01157}01157                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01158}01158 retry\_sleepq:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01159}01159                 \textcolor{keywordflow}{if} (((x \& SX\_LOCK\_WRITE\_SPINNER) \&\& SX\_SHARERS(x) == 0) ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01160}01160                     \mbox{\hyperlink{kern__sx_8c_a66ec52014dcfe61b9246cb5c1266dcad}{\_\_sx\_can\_read}}(td, x, \textcolor{keyword}{false})) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01161}01161                         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01162}01162                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01163}01163                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01164}01164 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01165}01165 \textcolor{preprocessor}{\#ifdef ADAPTIVE\_SX}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01166}01166                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01167}01167 \textcolor{comment}{                 * If the owner is running on another CPU, spin until}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01168}01168 \textcolor{comment}{                 * the owner stops running or the state of the lock}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01169}01169 \textcolor{comment}{                 * changes.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01170}01170 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01171}01171                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01172}01172                         owner = (\textcolor{keyword}{struct }thread *)SX\_OWNER(x);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01173}01173                         \textcolor{keywordflow}{if} (TD\_IS\_RUNNING(owner)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01174}01174                                 \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01175}01175                                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01176}01176                                 \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01177}01177                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01178}01178                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01179}01179 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01180}01180 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01181}01181                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01182}01182 \textcolor{comment}{                 * Try to set the SX\_LOCK\_SHARED\_WAITERS flag.  If we}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01183}01183 \textcolor{comment}{                 * fail to set it drop the sleep queue lock and loop}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01184}01184 \textcolor{comment}{                 * back.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01185}01185 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01186}01186                 \textcolor{keywordflow}{if} (!(x \& SX\_LOCK\_SHARED\_WAITERS)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01187}01187                         \textcolor{keywordflow}{if} (!atomic\_fcmpset\_ptr(\&sx-\/>sx\_lock, \&x,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01188}01188                             x | SX\_LOCK\_SHARED\_WAITERS))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01189}01189                                 \textcolor{keywordflow}{goto} retry\_sleepq;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01190}01190                         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01191}01191                                 CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p set shared waiters flag"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01192}01192                                     \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01193}01193                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01194}01194 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01195}01195                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01196}01196 \textcolor{comment}{                 * Since we have been unable to acquire the shared lock,}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01197}01197 \textcolor{comment}{                 * we have to sleep.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01198}01198 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01199}01199                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01200}01200                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p blocking on sleep queue"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01201}01201                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01202}01202 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01203}01203 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01204}01204                 sleep\_time -\/= \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01205}01205 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01206}01206                 \mbox{\hyperlink{subr__sleepqueue_8c_aa2822b5d83f7fb459964aa82b02b8701}{sleepq\_add}}(\&sx-\/>lock\_object, NULL, sx-\/>lock\_object.lo\_name,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01207}01207                     SLEEPQ\_SX | ((opts \& SX\_INTERRUPTIBLE) ?}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01208}01208                     SLEEPQ\_INTERRUPTIBLE : 0), \mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{SQ\_SHARED\_QUEUE}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01209}01209                 \textcolor{keywordflow}{if} (!(opts \& SX\_INTERRUPTIBLE))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01210}01210                         \mbox{\hyperlink{subr__sleepqueue_8c_ac839176dcd4b987ba04b3112b5502303}{sleepq\_wait}}(\&sx-\/>lock\_object, 0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01211}01211                 \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01212}01212                         error = \mbox{\hyperlink{subr__sleepqueue_8c_a1ceaeff8d86d1d875208f14e9342ded6}{sleepq\_wait\_sig}}(\&sx-\/>lock\_object, 0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01213}01213 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01214}01214                 sleep\_time += \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01215}01215                 sleep\_cnt++;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01216}01216 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01217}01217                 \textcolor{keywordflow}{if} (error) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01218}01218                         \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01219}01219                                 CTR2(KTR\_LOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01220}01220                         \textcolor{stringliteral}{"{}\%s: interruptible sleep by \%p suspended by signal"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01221}01221                                     \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01222}01222                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01223}01223                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01224}01224                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01225}01225                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p resuming from sleep queue"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01226}01226                             \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01227}01227                 x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01228}01228         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01229}01229 \textcolor{preprocessor}{\#if defined(KDTRACE\_HOOKS) || defined(LOCK\_PROFILING)}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01230}01230         \textcolor{keywordflow}{if} (\_\_predict\_true(!extra\_work))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01231}01231                 \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01232}01232 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01233}01233 \textcolor{preprocessor}{\#ifdef KDTRACE\_HOOKS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01234}01234         all\_time += \mbox{\hyperlink{kern__lockstat_8c_aa055f57fb00692f33e78feab4e7d80e8}{lockstat\_nsecs}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01235}01235         \textcolor{keywordflow}{if} (sleep\_time)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01236}01236                 LOCKSTAT\_RECORD4(sx\_\_block, sx, sleep\_time,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01237}01237                     LOCKSTAT\_READER, (state \& SX\_LOCK\_SHARED) == 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01238}01238                     (state \& SX\_LOCK\_SHARED) == 0 ? 0 : SX\_SHARERS(state));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01239}01239         \textcolor{keywordflow}{if} (lda.spin\_cnt > sleep\_cnt)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01240}01240                 LOCKSTAT\_RECORD4(sx\_\_spin, sx, all\_time -\/ sleep\_time,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01241}01241                     LOCKSTAT\_READER, (state \& SX\_LOCK\_SHARED) == 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01242}01242                     (state \& SX\_LOCK\_SHARED) == 0 ? 0 : SX\_SHARERS(state));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01243}01243 out\_lockstat:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01244}01244 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01245}01245         \textcolor{keywordflow}{if} (error == 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01246}01246                 LOCKSTAT\_PROFILE\_OBTAIN\_RWLOCK\_SUCCESS(sx\_\_acquire, sx,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01247}01247                     contested, waittime, file, line, LOCKSTAT\_READER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01248}01248         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01249}01249         \mbox{\hyperlink{kern__sx_8c_a4071a8f3605a24fcdac5e4cba9d0526f}{GIANT\_RESTORE}}();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01250}01250         \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01251}01251 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01252}01252 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01253}01253 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01254}\mbox{\hyperlink{kern__sx_8c_a358539e72da8dd6efa219dd8e33f9ff9}{01254}} \mbox{\hyperlink{kern__sx_8c_a358539e72da8dd6efa219dd8e33f9ff9}{\_sx\_slock\_int}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keywordtype}{int} opts LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01255}01255 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01256}01256         \textcolor{keyword}{struct }thread *td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01257}01257         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01258}01258         \textcolor{keywordtype}{int} error;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01259}01259 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01260}01260         KASSERT(\mbox{\hyperlink{subr__kdb_8c_a313e8ff6f20d39fd246ef5da5648c08c}{kdb\_active}} != 0 || SCHEDULER\_STOPPED() ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01261}01261             !TD\_IS\_IDLETHREAD(curthread),}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01262}01262             (\textcolor{stringliteral}{"{}sx\_slock() by idle thread \%p on sx \%s @ \%s:\%d"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01263}01263             curthread, sx-\/>lock\_object.lo\_name, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01264}01264         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01265}01265             (\textcolor{stringliteral}{"{}sx\_slock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01266}01266         WITNESS\_CHECKORDER(\&sx-\/>lock\_object, LOP\_NEWORDER, file, line, NULL);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01267}01267 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01268}01268         error = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01269}01269         td = curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01270}01270         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01271}01271         \textcolor{keywordflow}{if} (\_\_predict\_false(LOCKSTAT\_PROFILE\_ENABLED(sx\_\_acquire) ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01272}01272             !\mbox{\hyperlink{kern__sx_8c_a6ed0837b4bb3b770089354c33a1b3063}{\_\_sx\_slock\_try}}(sx, td, \&x, \textcolor{keyword}{true} LOCK\_FILE\_LINE\_ARG)))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01273}01273                 error = \mbox{\hyperlink{kern__sx_8c_a112785d75887c2f9917ba399e1a29c3f}{\_sx\_slock\_hard}}(sx, opts, x LOCK\_FILE\_LINE\_ARG);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01274}01274         \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01275}01275                 lock\_profile\_obtain\_lock\_success(\&sx-\/>lock\_object, \textcolor{keyword}{false}, 0, 0,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01276}01276                     file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01277}01277         \textcolor{keywordflow}{if} (error == 0) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01278}01278                 LOCK\_LOG\_LOCK(\textcolor{stringliteral}{"{}SLOCK"{}}, \&sx-\/>lock\_object, 0, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01279}01279                 WITNESS\_LOCK(\&sx-\/>lock\_object, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01280}01280                 TD\_LOCKS\_INC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01281}01281         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01282}01282         \textcolor{keywordflow}{return} (error);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01283}01283 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01284}01284 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01285}01285 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01286}\mbox{\hyperlink{kern__sx_8c_a7b05dba72622341e9a8301eb8d388577}{01286}} \mbox{\hyperlink{kern__sx_8c_a7b05dba72622341e9a8301eb8d388577}{\_sx\_slock}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keywordtype}{int} opts, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01287}01287 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01288}01288 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01289}01289         \textcolor{keywordflow}{return} (\mbox{\hyperlink{kern__sx_8c_a358539e72da8dd6efa219dd8e33f9ff9}{\_sx\_slock\_int}}(sx, opts LOCK\_FILE\_LINE\_ARG));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01290}01290 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01291}01291 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01292}01292 \textcolor{keyword}{static} \textcolor{keywordtype}{bool} \_\_always\_inline}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01293}\mbox{\hyperlink{kern__sx_8c_a81d688af1a27958688b946b17d0b409e}{01293}} \mbox{\hyperlink{kern__sx_8c_a81d688af1a27958688b946b17d0b409e}{\_sx\_sunlock\_try}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{struct} thread *td, uintptr\_t *xp)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01294}01294 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01295}01295 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01296}01296         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01297}01297                 \textcolor{keywordflow}{if} (SX\_SHARERS(*xp) > 1 || !(*xp \& SX\_LOCK\_WAITERS)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01298}01298                         \textcolor{keywordflow}{if} (atomic\_fcmpset\_rel\_ptr(\&sx-\/>sx\_lock, xp,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01299}01299                             *xp -\/ SX\_ONE\_SHARER)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01300}01300                                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01301}01301                                         CTR4(KTR\_LOCK,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01302}01302                                             \textcolor{stringliteral}{"{}\%s: \%p succeeded \%p -\/> \%p"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01303}01303                                             \_\_func\_\_, sx, (\textcolor{keywordtype}{void} *)*xp,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01304}01304                                             (\textcolor{keywordtype}{void} *)(*xp -\/ SX\_ONE\_SHARER));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01305}01305                                 td-\/>td\_sx\_slocks-\/-\/;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01306}01306                                 \textcolor{keywordflow}{return} (\textcolor{keyword}{true});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01307}01307                         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01308}01308                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01309}01309                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01310}01310                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01311}01311         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01312}01312         \textcolor{keywordflow}{return} (\textcolor{keyword}{false});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01313}01313 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01314}01314 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01315}01315 \textcolor{keyword}{static} \textcolor{keywordtype}{void} \_\_noinline}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01316}\mbox{\hyperlink{kern__sx_8c_a93205f1e2c3bfd6b7a5a4a64adfe6759}{01316}} \mbox{\hyperlink{kern__sx_8c_a93205f1e2c3bfd6b7a5a4a64adfe6759}{\_sx\_sunlock\_hard}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{struct} thread *td, uintptr\_t x}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01317}01317     LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01318}01318 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01319}01319         \textcolor{keywordtype}{int} wakeup\_swapper = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01320}01320         uintptr\_t setx, queue;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01321}01321 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01322}01322         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01323}01323                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01324}01324 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01325}01325         \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a81d688af1a27958688b946b17d0b409e}{\_sx\_sunlock\_try}}(sx, td, \&x))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01326}01326                 \textcolor{keywordflow}{goto} out\_lockstat;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01327}01327 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01328}01328         \mbox{\hyperlink{subr__sleepqueue_8c_ae0a612e63d885f5b440f1c0e84652baf}{sleepq\_lock}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01329}01329         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01330}01330         \textcolor{keywordflow}{for} (;;) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01331}01331                 \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a81d688af1a27958688b946b17d0b409e}{\_sx\_sunlock\_try}}(sx, td, \&x))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01332}01332                         \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01333}01333 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01334}01334                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01335}01335 \textcolor{comment}{                 * Wake up semantic here is quite simple:}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01336}01336 \textcolor{comment}{                 * Just wake up all the exclusive waiters.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01337}01337 \textcolor{comment}{                 * Note that the state of the lock could have changed,}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01338}01338 \textcolor{comment}{                 * so if it fails loop back and retry.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01339}01339 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01340}01340                 setx = SX\_LOCK\_UNLOCKED;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01341}01341                 queue = \mbox{\hyperlink{kern__sx_8c_a725a544f24cc0de84f6ad4e3f0c1e6b0}{SQ\_SHARED\_QUEUE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01342}01342                 \textcolor{keywordflow}{if} (x \& SX\_LOCK\_EXCLUSIVE\_WAITERS) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01343}01343                         setx |= (x \& SX\_LOCK\_SHARED\_WAITERS);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01344}01344                         queue = \mbox{\hyperlink{kern__sx_8c_a3cde1e96adc94f05ec2d174559d9cb8d}{SQ\_EXCLUSIVE\_QUEUE}};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01345}01345                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01346}01346                 setx |= (x \& SX\_LOCK\_WRITE\_SPINNER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01347}01347                 \textcolor{keywordflow}{if} (!atomic\_fcmpset\_rel\_ptr(\&sx-\/>sx\_lock, \&x, setx))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01348}01348                         \textcolor{keywordflow}{continue};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01349}01349                 \textcolor{keywordflow}{if} (LOCK\_LOG\_TEST(\&sx-\/>lock\_object, 0))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01350}01350                         CTR2(KTR\_LOCK, \textcolor{stringliteral}{"{}\%s: \%p waking up all thread on"{}}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01351}01351                             \textcolor{stringliteral}{"{}exclusive queue"{}}, \_\_func\_\_, sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01352}01352                 wakeup\_swapper = \mbox{\hyperlink{subr__sleepqueue_8c_a2c739ee96a9938ed1de78a8772311bd7}{sleepq\_broadcast}}(\&sx-\/>lock\_object, SLEEPQ\_SX,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01353}01353                     0, queue);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01354}01354                 td-\/>td\_sx\_slocks-\/-\/;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01355}01355                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01356}01356         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01357}01357         \mbox{\hyperlink{subr__sleepqueue_8c_a0157460dbeca34582ee4d5890b74fa15}{sleepq\_release}}(\&sx-\/>lock\_object);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01358}01358         \textcolor{keywordflow}{if} (wakeup\_swapper)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01359}01359                 kick\_proc0();}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01360}01360 out\_lockstat:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01361}01361         LOCKSTAT\_PROFILE\_RELEASE\_RWLOCK(sx\_\_release, sx, LOCKSTAT\_READER);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01362}01362 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01363}01363 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01364}01364 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01365}\mbox{\hyperlink{kern__sx_8c_aa0f8e781e61e868642aa469f791b4ebf}{01365}} \mbox{\hyperlink{kern__sx_8c_aa0f8e781e61e868642aa469f791b4ebf}{\_sx\_sunlock\_int}}(\textcolor{keyword}{struct} sx *sx LOCK\_FILE\_LINE\_ARG\_DEF)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01366}01366 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01367}01367         \textcolor{keyword}{struct }thread *td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01368}01368         uintptr\_t x;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01369}01369 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01370}01370         KASSERT(sx-\/>sx\_lock != SX\_LOCK\_DESTROYED,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01371}01371             (\textcolor{stringliteral}{"{}sx\_sunlock() of destroyed sx @ \%s:\%d"{}}, file, line));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01372}01372         \mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{\_sx\_assert}}(sx, SA\_SLOCKED, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01373}01373         WITNESS\_UNLOCK(\&sx-\/>lock\_object, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01374}01374         LOCK\_LOG\_LOCK(\textcolor{stringliteral}{"{}SUNLOCK"{}}, \&sx-\/>lock\_object, 0, 0, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01375}01375 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01376}01376         td = curthread;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01377}01377         x = SX\_READ\_VALUE(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01378}01378         \textcolor{keywordflow}{if} (\_\_predict\_false(LOCKSTAT\_PROFILE\_ENABLED(sx\_\_release) ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01379}01379             !\mbox{\hyperlink{kern__sx_8c_a81d688af1a27958688b946b17d0b409e}{\_sx\_sunlock\_try}}(sx, td, \&x)))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01380}01380                 \mbox{\hyperlink{kern__sx_8c_a93205f1e2c3bfd6b7a5a4a64adfe6759}{\_sx\_sunlock\_hard}}(sx, td, x LOCK\_FILE\_LINE\_ARG);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01381}01381         \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01382}01382                 lock\_profile\_release\_lock(\&sx-\/>lock\_object, \textcolor{keyword}{false});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01383}01383 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01384}01384         TD\_LOCKS\_DEC(curthread);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01385}01385 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01386}01386 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01387}01387 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01388}\mbox{\hyperlink{kern__sx_8c_a860700a0361b4749d75c166ea328a62b}{01388}} \mbox{\hyperlink{kern__sx_8c_a860700a0361b4749d75c166ea328a62b}{\_sx\_sunlock}}(\textcolor{keyword}{struct} sx *sx, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01389}01389 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01390}01390 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01391}01391         \mbox{\hyperlink{kern__sx_8c_aa0f8e781e61e868642aa469f791b4ebf}{\_sx\_sunlock\_int}}(sx LOCK\_FILE\_LINE\_ARG);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01392}01392 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01393}01393 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01394}01394 \textcolor{preprocessor}{\#ifdef INVARIANT\_SUPPORT}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01395}01395 \textcolor{preprocessor}{\#ifndef INVARIANTS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01396}01396 \textcolor{preprocessor}{\#undef  \_sx\_assert}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01397}01397 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01398}01398 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01399}01399 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01400}01400 \textcolor{comment}{ * In the non-\/WITNESS case, sx\_assert() can only detect that at least}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01401}01401 \textcolor{comment}{ * *some* thread owns an slock, but it cannot guarantee that *this*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01402}01402 \textcolor{comment}{ * thread owns an slock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01403}01403 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01404}01404 \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01405}01405 \mbox{\hyperlink{kern__sx_8c_a0478335af0f3e1c3551f24f5a15f4af0}{\_sx\_assert}}(\textcolor{keyword}{const} \textcolor{keyword}{struct} sx *sx, \textcolor{keywordtype}{int} what, \textcolor{keyword}{const} \textcolor{keywordtype}{char} *file, \textcolor{keywordtype}{int} line)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01406}01406 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01407}01407 \textcolor{preprocessor}{\#ifndef WITNESS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01408}01408         \textcolor{keywordtype}{int} slocked = 0;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01409}01409 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01410}01410 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01411}01411         \textcolor{keywordflow}{if} (SCHEDULER\_STOPPED())}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01412}01412                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01413}01413         \textcolor{keywordflow}{switch} (what) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01414}01414         \textcolor{keywordflow}{case} SA\_SLOCKED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01415}01415         \textcolor{keywordflow}{case} SA\_SLOCKED | SA\_NOTRECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01416}01416         \textcolor{keywordflow}{case} SA\_SLOCKED | SA\_RECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01417}01417 \textcolor{preprocessor}{\#ifndef WITNESS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01418}01418                 slocked = 1;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01419}01419                 \textcolor{comment}{/* FALLTHROUGH */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01420}01420 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01421}01421         \textcolor{keywordflow}{case} SA\_LOCKED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01422}01422         \textcolor{keywordflow}{case} SA\_LOCKED | SA\_NOTRECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01423}01423         \textcolor{keywordflow}{case} SA\_LOCKED | SA\_RECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01424}01424 \textcolor{preprocessor}{\#ifdef WITNESS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01425}01425                 \mbox{\hyperlink{subr__witness_8c_a2bfba169d4d1cf8a6d574e11e4fa075b}{witness\_assert}}(\&sx-\/>lock\_object, what, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01426}01426 \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01427}01427                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01428}01428 \textcolor{comment}{                 * If some other thread has an exclusive lock or we}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01429}01429 \textcolor{comment}{                 * have one and are asserting a shared lock, fail.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01430}01430 \textcolor{comment}{                 * Also, if no one has a lock at all, fail.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01431}01431 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01432}01432                 \textcolor{keywordflow}{if} (sx-\/>sx\_lock == SX\_LOCK\_UNLOCKED ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01433}01433                     (!(sx-\/>sx\_lock \& SX\_LOCK\_SHARED) \&\& (slocked ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01434}01434                     sx\_xholder(sx) != curthread)))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01435}01435                         \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s not \%slocked @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01436}01436                             sx-\/>lock\_object.lo\_name, slocked ? \textcolor{stringliteral}{"{}share "{}} : \textcolor{stringliteral}{"{}"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01437}01437                             file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01438}01438 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01439}01439                 \textcolor{keywordflow}{if} (!(sx-\/>sx\_lock \& SX\_LOCK\_SHARED)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01440}01440                         \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a5745cf1e8f892977266d7fd5d04dbe2b}{sx\_recursed}}(sx)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01441}01441                                 \textcolor{keywordflow}{if} (what \& SA\_NOTRECURSED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01442}01442                                         \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s recursed @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01443}01443                                             sx-\/>lock\_object.lo\_name, file,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01444}01444                                             line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01445}01445                         \} \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (what \& SA\_RECURSED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01446}01446                                 \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s not recursed @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01447}01447                                     sx-\/>lock\_object.lo\_name, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01448}01448                 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01449}01449 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01450}01450                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01451}01451         \textcolor{keywordflow}{case} SA\_XLOCKED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01452}01452         \textcolor{keywordflow}{case} SA\_XLOCKED | SA\_NOTRECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01453}01453         \textcolor{keywordflow}{case} SA\_XLOCKED | SA\_RECURSED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01454}01454                 \textcolor{keywordflow}{if} (sx\_xholder(sx) != curthread)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01455}01455                         \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s not exclusively locked @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01456}01456                             sx-\/>lock\_object.lo\_name, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01457}01457                 \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a5745cf1e8f892977266d7fd5d04dbe2b}{sx\_recursed}}(sx)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01458}01458                         \textcolor{keywordflow}{if} (what \& SA\_NOTRECURSED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01459}01459                                 \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s recursed @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01460}01460                                     sx-\/>lock\_object.lo\_name, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01461}01461                 \} \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (what \& SA\_RECURSED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01462}01462                         \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s not recursed @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01463}01463                             sx-\/>lock\_object.lo\_name, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01464}01464                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01465}01465         \textcolor{keywordflow}{case} SA\_UNLOCKED:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01466}01466 \textcolor{preprocessor}{\#ifdef WITNESS}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01467}01467                 \mbox{\hyperlink{subr__witness_8c_a2bfba169d4d1cf8a6d574e11e4fa075b}{witness\_assert}}(\&sx-\/>lock\_object, what, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01468}01468 \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01469}01469                 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01470}01470 \textcolor{comment}{                 * If we hold an exclusve lock fail.  We can't}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01471}01471 \textcolor{comment}{                 * reliably check to see if we hold a shared lock or}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01472}01472 \textcolor{comment}{                 * not.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01473}01473 \textcolor{comment}{                 */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01474}01474                 \textcolor{keywordflow}{if} (sx\_xholder(sx) == curthread)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01475}01475                         \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Lock \%s exclusively locked @ \%s:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01476}01476                             sx-\/>lock\_object.lo\_name, file, line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01477}01477 \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01478}01478                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01479}01479         \textcolor{keywordflow}{default}:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01480}01480                 \mbox{\hyperlink{kern__shutdown_8c_ad1f1a1dc0581a11b597453f148106393}{panic}}(\textcolor{stringliteral}{"{}Unknown sx lock assertion: \%d @ \%s:\%d"{}}, what, file,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01481}01481                     line);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01482}01482         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01483}01483 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01484}01484 \textcolor{preprocessor}{\#endif  }\textcolor{comment}{/* INVARIANT\_SUPPORT */}\textcolor{preprocessor}{}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01485}01485 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01486}01486 \textcolor{preprocessor}{\#ifdef DDB}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01487}01487 \textcolor{keyword}{static} \textcolor{keywordtype}{void}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01488}01488 db\_show\_sx(\textcolor{keyword}{const} \textcolor{keyword}{struct} lock\_object *lock)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01489}01489 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01490}01490         \textcolor{keyword}{struct }thread *td;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01491}01491         \textcolor{keyword}{const} \textcolor{keyword}{struct }sx *sx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01492}01492 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01493}01493         sx = (\textcolor{keyword}{const} \textcolor{keyword}{struct }sx *)lock;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01494}01494 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01495}01495         db\_printf(\textcolor{stringliteral}{"{} state: "{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01496}01496         \textcolor{keywordflow}{if} (sx-\/>sx\_lock == SX\_LOCK\_UNLOCKED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01497}01497                 db\_printf(\textcolor{stringliteral}{"{}UNLOCKED\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01498}01498         \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (sx-\/>sx\_lock == SX\_LOCK\_DESTROYED) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01499}01499                 db\_printf(\textcolor{stringliteral}{"{}DESTROYED\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01500}01500                 \textcolor{keywordflow}{return};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01501}01501         \} \textcolor{keywordflow}{else} \textcolor{keywordflow}{if} (sx-\/>sx\_lock \& SX\_LOCK\_SHARED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01502}01502                 db\_printf(\textcolor{stringliteral}{"{}SLOCK: \%ju\(\backslash\)n"{}}, (uintmax\_t)SX\_SHARERS(sx-\/>sx\_lock));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01503}01503         \textcolor{keywordflow}{else} \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01504}01504                 td = sx\_xholder(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01505}01505                 db\_printf(\textcolor{stringliteral}{"{}XLOCK: \%p (tid \%d, pid \%d, \(\backslash\)"{}\%s\(\backslash\)"{})\(\backslash\)n"{}}, td,}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01506}01506                     td-\/>td\_tid, td-\/>td\_proc-\/>p\_pid, td-\/>td\_name);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01507}01507                 \textcolor{keywordflow}{if} (\mbox{\hyperlink{kern__sx_8c_a5745cf1e8f892977266d7fd5d04dbe2b}{sx\_recursed}}(sx))}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01508}01508                         db\_printf(\textcolor{stringliteral}{"{} recursed: \%d\(\backslash\)n"{}}, sx-\/>sx\_recurse);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01509}01509         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01510}01510 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01511}01511         db\_printf(\textcolor{stringliteral}{"{} waiters: "{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01512}01512         \textcolor{keywordflow}{switch}(sx-\/>sx\_lock \&}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01513}01513             (SX\_LOCK\_SHARED\_WAITERS | SX\_LOCK\_EXCLUSIVE\_WAITERS)) \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01514}01514         \textcolor{keywordflow}{case} SX\_LOCK\_SHARED\_WAITERS:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01515}01515                 db\_printf(\textcolor{stringliteral}{"{}shared\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01516}01516                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01517}01517         \textcolor{keywordflow}{case} SX\_LOCK\_EXCLUSIVE\_WAITERS:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01518}01518                 db\_printf(\textcolor{stringliteral}{"{}exclusive\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01519}01519                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01520}01520         \textcolor{keywordflow}{case} SX\_LOCK\_SHARED\_WAITERS | SX\_LOCK\_EXCLUSIVE\_WAITERS:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01521}01521                 db\_printf(\textcolor{stringliteral}{"{}exclusive and shared\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01522}01522                 \textcolor{keywordflow}{break};}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01523}01523         \textcolor{keywordflow}{default}:}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01524}01524                 db\_printf(\textcolor{stringliteral}{"{}none\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01525}01525         \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01526}01526 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01527}01527 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01528}01528 \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01529}01529 \textcolor{comment}{ * Check to see if a thread that is blocked on a sleep queue is actually}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01530}01530 \textcolor{comment}{ * blocked on an sx lock.  If so, output some details and return true.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01531}01531 \textcolor{comment}{ * If the lock has an exclusive owner, return that in *ownerp.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01532}01532 \textcolor{comment}{ */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01533}01533 \textcolor{keywordtype}{int}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01534}01534 sx\_chain(\textcolor{keyword}{struct} thread *td, \textcolor{keyword}{struct} thread **ownerp)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01535}01535 \{}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01536}01536         \textcolor{keyword}{const} \textcolor{keyword}{struct }sx *sx;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01537}01537 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01538}01538         \textcolor{comment}{/*}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01539}01539 \textcolor{comment}{         * Check to see if this thread is blocked on an sx lock.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01540}01540 \textcolor{comment}{         * First, we check the lock class.  If that is ok, then we}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01541}01541 \textcolor{comment}{         * compare the lock name against the wait message.}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01542}01542 \textcolor{comment}{         */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01543}01543         sx = td-\/>td\_wchan;}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01544}01544         \textcolor{keywordflow}{if} (LOCK\_CLASS(\&sx-\/>lock\_object) != \&\mbox{\hyperlink{kern__sx_8c_a188613c956d39c9b944b9de278ce2801}{lock\_class\_sx}} ||}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01545}01545             sx-\/>lock\_object.lo\_name != td-\/>td\_wmesg)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01546}01546                 \textcolor{keywordflow}{return} (0);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01547}01547 }
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01548}01548         \textcolor{comment}{/* We think we have an sx lock, so output some details. */}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01549}01549         db\_printf(\textcolor{stringliteral}{"{}blocked on sx \(\backslash\)"{}\%s\(\backslash\)"{} "{}}, td-\/>td\_wmesg);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01550}01550         *ownerp = sx\_xholder(sx);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01551}01551         \textcolor{keywordflow}{if} (sx-\/>sx\_lock \& SX\_LOCK\_SHARED)}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01552}01552                 db\_printf(\textcolor{stringliteral}{"{}SLOCK (count \%ju)\(\backslash\)n"{}},}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01553}01553                     (uintmax\_t)SX\_SHARERS(sx-\/>sx\_lock));}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01554}01554         \textcolor{keywordflow}{else}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01555}01555                 db\_printf(\textcolor{stringliteral}{"{}XLOCK\(\backslash\)n"{}});}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01556}01556         \textcolor{keywordflow}{return} (1);}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01557}01557 \}}
\DoxyCodeLine{\Hypertarget{kern__sx_8c_source_l01558}01558 \textcolor{preprocessor}{\#endif}}

\end{DoxyCode}
