1 #include <linux/compiler.h>
2 #include <linux/module.h>
3 #include <linux/types.h>
4
5 #include <asm/processor.h>
6 #include <asm/cmpxchg.h>
7 #include <asm/atomic.h>
8
9 static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new)
10 {
11 u32 low = new;
12 u32 high = new >> 32;
13
14 asm volatile(
15 LOCK_PREFIX "cmpxchg8b %1\n"
16 : "+A" (old), "+m" (*ptr)
17 : "b" (low), "c" (high)
18 );
19 return old;
20 }
21
22 u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val)
23 {
24 return cmpxchg8b(&ptr->counter, old_val, new_val);
25 }
26 EXPORT_SYMBOL(atomic64_cmpxchg);
27
28 /**
29 * atomic64_xchg - xchg atomic64 variable
30 * @ptr: pointer to type atomic64_t
31 * @new_val: value to assign
32 *
33 * Atomically xchgs the value of @ptr to @new_val and returns
34 * the old value.
35 */
36 u64 atomic64_xchg(atomic64_t *ptr, u64 new_val)
37 {
38 /*
39 * Try first with a (possibly incorrect) assumption about
40 * what we have there. We'll do two loops most likely,
41 * but we'll get an ownership MESI transaction straight away
42 * instead of a read transaction followed by a
43 * flush-for-ownership transaction:
44 */
45 u64 old_val, real_val = 0;
46
47 do {
48 old_val = real_val;
49
50 real_val = atomic64_cmpxchg(ptr, old_val, new_val);
51
52 } while (real_val != old_val);
53
54 return old_val;
55 }
56 EXPORT_SYMBOL(atomic64_xchg);
57
58 /**
59 * atomic64_set - set atomic64 variable
60 * @ptr: pointer to type atomic64_t
61 * @new_val: value to assign
62 *
63 * Atomically sets the value of @ptr to @new_val.
64 */
65 void atomic64_set(atomic64_t *ptr, u64 new_val)
66 {
67 atomic64_xchg(ptr, new_val);
68 }
69 EXPORT_SYMBOL(atomic64_set);
70
71 /**
72 EXPORT_SYMBOL(atomic64_read);
73 * atomic64_add_return - add and return
74 * @delta: integer value to add
75 * @ptr: pointer to type atomic64_t
76 *
77 * Atomically adds @delta to @ptr and returns @delta + *@ptr
78 */
79 noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr)
80 {
81 /*
82 * Try first with a (possibly incorrect) assumption about
83 * what we have there. We'll do two loops most likely,
84 * but we'll get an ownership MESI transaction straight away
85 * instead of a read transaction followed by a
86 * flush-for-ownership transaction:
87 */
88 u64 old_val, new_val, real_val = 0;
89
90 do {
91 old_val = real_val;
92 new_val = old_val + delta;
93
94 real_val = atomic64_cmpxchg(ptr, old_val, new_val);
95
96 } while (real_val != old_val);
97
98 return new_val;
99 }
100 EXPORT_SYMBOL(atomic64_add_return);
101
102 u64 atomic64_sub_return(u64 delta, atomic64_t *ptr)
103 {
104 return atomic64_add_return(-delta, ptr);
105 }
106 EXPORT_SYMBOL(atomic64_sub_return);
107
108 u64 atomic64_inc_return(atomic64_t *ptr)
109 {
110 return atomic64_add_return(1, ptr);
111 }
112 EXPORT_SYMBOL(atomic64_inc_return);
113
114 u64 atomic64_dec_return(atomic64_t *ptr)
115 {
116 return atomic64_sub_return(1, ptr);
117 }
118 EXPORT_SYMBOL(atomic64_dec_return);
119
120 /**
121 * atomic64_add - add integer to atomic64 variable
122 * @delta: integer value to add
123 * @ptr: pointer to type atomic64_t
124 *
125 * Atomically adds @delta to @ptr.
126 */
127 void atomic64_add(u64 delta, atomic64_t *ptr)
128 {
129 atomic64_add_return(delta, ptr);
130 }
131 EXPORT_SYMBOL(atomic64_add);
132
133 /**
134 * atomic64_sub - subtract the atomic64 variable
135 * @delta: integer value to subtract
136 * @ptr: pointer to type atomic64_t
137 *
138 * Atomically subtracts @delta from @ptr.
139 */
140 void atomic64_sub(u64 delta, atomic64_t *ptr)
141 {
142 atomic64_add(-delta, ptr);
143 }
144 EXPORT_SYMBOL(atomic64_sub);
145
146 /**
147 * atomic64_sub_and_test - subtract value from variable and test result
148 * @delta: integer value to subtract
149 * @ptr: pointer to type atomic64_t
150 *
151 * Atomically subtracts @delta from @ptr and returns
152 * true if the result is zero, or false for all
153 * other cases.
154 */
155 int atomic64_sub_and_test(u64 delta, atomic64_t *ptr)
156 {
157 u64 new_val = atomic64_sub_return(delta, ptr);
158
159 return new_val == 0;
160 }
161 EXPORT_SYMBOL(atomic64_sub_and_test);
162
163 /**
164 * atomic64_inc - increment atomic64 variable
165 * @ptr: pointer to type atomic64_t
166 *
167 * Atomically increments @ptr by 1.
168 */
169 void atomic64_inc(atomic64_t *ptr)
170 {
171 atomic64_add(1, ptr);
172 }
173 EXPORT_SYMBOL(atomic64_inc);
174
175 /**
176 * atomic64_dec - decrement atomic64 variable
177 * @ptr: pointer to type atomic64_t
178 *
179 * Atomically decrements @ptr by 1.
180 */
181 void atomic64_dec(atomic64_t *ptr)
182 {
183 atomic64_sub(1, ptr);
184 }
185 EXPORT_SYMBOL(atomic64_dec);
186
187 /**
188 * atomic64_dec_and_test - decrement and test
189 * @ptr: pointer to type atomic64_t
190 *
191 * Atomically decrements @ptr by 1 and
192 * returns true if the result is 0, or false for all other
193 * cases.
194 */
195 int atomic64_dec_and_test(atomic64_t *ptr)
196 {
197 return atomic64_sub_and_test(1, ptr);
198 }
199 EXPORT_SYMBOL(atomic64_dec_and_test);
200
201 /**
202 * atomic64_inc_and_test - increment and test
203 * @ptr: pointer to type atomic64_t
204 *
205 * Atomically increments @ptr by 1
206 * and returns true if the result is zero, or false for all
207 * other cases.
208 */
209 int atomic64_inc_and_test(atomic64_t *ptr)
210 {
211 return atomic64_sub_and_test(-1, ptr);
212 }
213 EXPORT_SYMBOL(atomic64_inc_and_test);
214
215 /**
216 * atomic64_add_negative - add and test if negative
217 * @delta: integer value to add
218 * @ptr: pointer to type atomic64_t
219 *
220 * Atomically adds @delta to @ptr and returns true
221 * if the result is negative, or false when
222 * result is greater than or equal to zero.
223 */
224 int atomic64_add_negative(u64 delta, atomic64_t *ptr)
225 {
226 s64 new_val = atomic64_add_return(delta, ptr);
227
228 return new_val < 0;
229 }
230 EXPORT_SYMBOL(atomic64_add_negative);
231
|
This page was automatically generated by the
LXR engine.
|