]>
git.proxmox.com Git - ceph.git/blob - ceph/src/brag/client/ceph-brag
3 from __future__
import print_function
12 from operator
import itemgetter
13 from heapq
import nlargest
16 CLUSTER_UUID_NAME
='cluster-uuid'
17 CLUSTER_OWNERSHIP_NAME
='cluster-ownership'
23 from collections
import Counter
25 from itertools
import repeat
, ifilter
28 '''Dict subclass for counting hashable objects. Sometimes called a bag
29 or multiset. Elements are stored as dictionary keys and their counts
30 are stored as dictionary values.
33 Counter({'y': 3, 'z': 2, 'g': 1})
37 def __init__(self
, iterable
=None, **kwds
):
38 '''Create a new, empty Counter object. And if given, count elements
39 from an input iterable. Or, initialize the count from another mapping
40 of elements to their counts.
42 >>> c = Counter() # a new, empty counter
43 >>> c = Counter('gallahad') # a new counter from an iterable
44 >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
45 >>> c = Counter(a=4, b=2) # a new counter from keyword args
48 self
.update(iterable
, **kwds
)
50 def __missing__(self
, key
):
53 def most_common(self
, n
=None):
54 '''List the n most common elements and their counts from the most
55 common to the least. If n is None, then list all element counts.
57 >>> Counter('abracadabra').most_common(3)
58 [('a', 5), ('r', 2), ('b', 2)]
62 return sorted(self
.iteritems(), key
=itemgetter(1), reverse
=True)
63 return nlargest(n
, self
.iteritems(), key
=itemgetter(1))
66 '''Iterator over elements repeating each as many times as its count.
68 >>> c = Counter('ABCABC')
69 >>> sorted(c.elements())
70 ['A', 'A', 'B', 'B', 'C', 'C']
72 If an element's count has been set to zero or is a negative number,
73 elements() will ignore it.
76 for elem
, count
in self
.iteritems():
77 for _
in repeat(None, count
):
80 # Override dict methods where the meaning changes for Counter objects.
83 def fromkeys(cls
, iterable
, v
=None):
84 raise NotImplementedError(
85 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
87 def update(self
, iterable
=None, **kwds
):
88 '''Like dict.update() but add counts instead of replacing them.
90 Source can be an iterable, a dictionary, or another Counter instance.
92 >>> c = Counter('which')
93 >>> c.update('witch') # add elements from another iterable
94 >>> d = Counter('watch')
95 >>> c.update(d) # add elements from another counter
96 >>> c['h'] # four 'h' in which, witch, and watch
100 if iterable
is not None:
101 if hasattr(iterable
, 'iteritems'):
104 for elem
, count
in iterable
.iteritems():
105 self
[elem
] = self_get(elem
, 0) + count
107 dict.update(self
, iterable
) # fast path when counter is empty
110 for elem
in iterable
:
111 self
[elem
] = self_get(elem
, 0) + 1
116 'Like dict.copy() but returns a Counter instance instead of a dict.'
119 def __delitem__(self
, elem
):
120 'Like dict.__delitem__() but does not raise KeyError for missing values.'
122 dict.__delitem
__(self
, elem
)
126 return '%s()' % self
.__class
__.__name
__
127 items
= ', '.join(map('%r: %r'.__mod
__, self
.most_common()))
128 return '%s({%s})' % (self
.__class
__.__name
__, items
)
130 # Multiset-style mathematical operations discussed in:
131 # Knuth TAOCP Volume II section 4.6.3 exercise 19
132 # and at http://en.wikipedia.org/wiki/Multiset
134 # Outputs guaranteed to only include positive counts.
136 # To strip negative and zero counts, add-in an empty counter:
139 def __add__(self
, other
):
140 '''Add counts from two counters.
142 >>> Counter('abbb') + Counter('bcc')
143 Counter({'b': 4, 'c': 2, 'a': 1})
147 if not isinstance(other
, Counter
):
148 return NotImplemented
150 for elem
in set(self
) |
set(other
):
151 newcount
= self
[elem
] + other
[elem
]
153 result
[elem
] = newcount
156 def __sub__(self
, other
):
157 ''' Subtract count, but keep only results with positive counts.
159 >>> Counter('abbbc') - Counter('bccd')
160 Counter({'b': 2, 'a': 1})
163 if not isinstance(other
, Counter
):
164 return NotImplemented
166 for elem
in set(self
) |
set(other
):
167 newcount
= self
[elem
] - other
[elem
]
169 result
[elem
] = newcount
172 def __or__(self
, other
):
173 '''Union is the maximum of value in either of the input counters.
175 >>> Counter('abbb') | Counter('bcc')
176 Counter({'b': 3, 'c': 2, 'a': 1})
179 if not isinstance(other
, Counter
):
180 return NotImplemented
183 for elem
in set(self
) |
set(other
):
184 newcount
= _max(self
[elem
], other
[elem
])
186 result
[elem
] = newcount
189 def __and__(self
, other
):
190 ''' Intersection is the minimum of corresponding counts.
192 >>> Counter('abbb') & Counter('bcc')
196 if not isinstance(other
, Counter
):
197 return NotImplemented
200 if len(self
) < len(other
):
201 self
, other
= other
, self
202 for elem
in ifilter(self
.__contains
__, other
):
203 newcount
= _min(self
[elem
], other
[elem
])
205 result
[elem
] = newcount
209 def print_stderr(*args
, **kwargs
):
210 kwargs
.setdefault('file', sys
.stderr
)
211 print(*args
, **kwargs
)
213 def run_command(cmd
):
215 print_stderr("run_command: " + str(cmd
))
216 child
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
,
217 stderr
=subprocess
.PIPE
)
218 (o
, e
) = child
.communicate()
219 o
= o
.decode('utf-8', 'ignore')
220 e
= e
.decode('utf-8', 'ignore')
221 return (child
.returncode
, o
, e
)
225 (rc
,uid
,e
) = run_command(['ceph', 'config-key', 'get', CLUSTER_UUID_NAME
])
227 #uuid is not yet set.
228 uid
= str(uuid
.uuid4())
229 (rc
, o
, e
) = run_command(['ceph', 'config-key', 'set',
230 CLUSTER_UUID_NAME
, uid
])
232 raise RuntimeError("\'ceph config-key set\' failed -" + e
)
236 def bytes_pretty_to_raw(byte_count
, byte_scale
):
237 if byte_scale
== 'kB':
238 return byte_count
>> 10
239 if byte_scale
== 'MB':
240 return byte_count
>> 20
241 if byte_scale
== 'GB':
242 return byte_count
>> 30
243 if byte_scale
== 'TB':
244 return byte_count
>> 40
245 if byte_scale
== 'PB':
246 return byte_count
>> 50
247 if byte_scale
== 'EB':
248 return byte_count
>> 60
253 (rc
, o
, e
) = run_command(['ceph', '-s', '-f', 'json'])
255 raise RuntimeError("\'ceph -s\' failed - " + e
)
258 num_mons
= len(oj
['monmap']['mons'])
259 num_osds
= int(oj
['osdmap']['osdmap']['num_in_osds'])
261 num_mdss
= oj
['fsmap']['in']
266 num_pgs
= pgmap
['num_pgs']
267 num_data_bytes
= pgmap
['data_bytes']
268 num_bytes_total
= pgmap
['bytes_total']
270 (rc
, o
, e
) = run_command(['ceph', 'pg', 'dump', 'pools', '-f', 'json-pretty'])
272 raise RuntimeError("\'ceph pg dump pools\' failed - " + e
)
274 pools
= json
.loads(o
)
275 num_pools
= len(pools
)
278 num_objs
+= p
['stat_sum']['num_objects']
280 nums
= {'num_mons':num_mons
,
284 'num_data_bytes':num_data_bytes
,
285 'num_bytes_total':num_bytes_total
,
286 'num_pools':num_pools
,
287 'num_objects':num_objs
}
290 def get_crush_types():
291 (rc
, o
, e
) = run_command(['ceph', 'osd', 'crush', 'dump'])
293 raise RuntimeError("\'ceph osd crush dump\' failed - " + e
)
295 crush_dump
= json
.loads(o
)
296 if crush_dump
['types'] is None:
297 raise RuntimeError("\'types\' item missing in \'ceph osd crush dump\'")
300 for t
in crush_dump
['types']:
301 crush_types
[t
['type_id']] = t
['name']
304 for bucket
in crush_dump
['buckets']:
305 types_list
.append(bucket
['type_id'])
308 types_counter
= Counter(types_list
)
309 append
= lambda t
,c
: crush_map
.append({'type':t
, 'count':c
})
310 for id,count
in types_counter
.items():
311 append(crush_types
[id],
314 if 'devices' in crush_dump
:
315 append('devices', len(crush_dump
['devices']))
319 def get_osd_dump_info():
320 (rc
, o
, e
) = run_command(['ceph', 'osd', 'dump', '-f', 'json'])
322 raise RuntimeError("\'ceph osd dump\' failed - " + e
)
326 proc
= lambda x
: {'id':x
['pool'], 'type':x
['type'], 'size':x
['size']}
327 for p
in oj
['pools']:
328 pool_meta
.append(proc(p
))
330 return oj
['created'], pool_meta
332 def get_sysinfo(max_osds
):
334 osd_metadata_available
= False
338 kern_description
= {}
344 incr
= lambda a
,k
: 1 if k
not in a
else a
[k
]+1
345 while count
< max_osds
:
346 (rc
, o
, e
) = run_command(['ceph', 'osd', 'metadata', str(count
)])
348 if not osd_metadata_available
:
349 osd_metadata_available
= True
351 jmeta
= json
.loads(o
)
353 version
= jmeta
['ceph_version'].split()
355 if (len(version
) > 3):
358 ceph_version
[cv
] = incr(ceph_version
, cv
)
359 os
[jmeta
['os']] = incr(os
, jmeta
['os'])
360 kern_version
[jmeta
['kernel_version']] = \
361 incr(kern_version
, jmeta
['kernel_version'])
362 kern_description
[jmeta
['kernel_description']] = \
363 incr(kern_description
, jmeta
['kernel_description'])
366 dstr
= jmeta
['distro'] + ' '
367 dstr
+= jmeta
['distro_version'] + ' '
368 dstr
+= jmeta
['distro_codename'] + ' ('
369 dstr
+= jmeta
['distro_description'] + ')'
370 distro
[dstr
] = incr(distro
, dstr
)
374 cpu
[jmeta
['cpu']] = incr(cpu
, jmeta
['cpu'])
375 arch
[jmeta
['arch']] = incr(arch
, jmeta
['arch'])
380 if not osd_metadata_available
:
381 print_stderr("'ceph osd metadata' is not available at all")
384 def jsonify(type_count
, name
, type_name
):
386 for k
, v
in type_count
.items():
387 tmp
.append({type_name
:k
, 'count':v
})
390 jsonify(os
, 'os_info', 'os')
391 jsonify(kern_version
, 'kernel_versions', 'version')
392 jsonify(kern_description
, 'kernel_types', 'type')
393 jsonify(distro
, 'distros', 'distro')
394 jsonify(cpu
, 'cpus', 'cpu')
395 jsonify(arch
, 'cpu_archs', 'arch')
396 jsonify(ceph_version
, 'ceph_versions', 'version')
399 def get_ownership_info():
400 (rc
, o
, e
) = run_command(['ceph', 'config-key', 'get',
401 CLUSTER_OWNERSHIP_NAME
])
405 return ast
.literal_eval(o
)
411 out
['uuid'] = get_uuid()
413 num_osds
= int(nums
['num_osds'])
414 out
['components_count'] = nums
415 out
['crush_types'] = get_crush_types()
416 out
['cluster_creation_date'], out
['pool_metadata'] = get_osd_dump_info()
417 out
['sysinfo'] = get_sysinfo(num_osds
)
419 owner
= get_ownership_info()
420 if owner
is not None:
421 out
['ownership'] = owner
423 url
= owner
.pop('url')
425 return json
.dumps(out
, indent
=2, separators
=(',', ': ')), url
427 def describe_usage():
428 print_stderr("Usage:")
429 print_stderr("======")
431 print_stderr(sys
.argv
[0] + " [-v|--verbose] [<commands> [command-options]]")
433 print_stderr("without any option, shows the data to be published and do nothing")
435 print_stderr("-v|--verbose: toggle verbose output on stdout")
437 print_stderr("commands:")
438 print_stderr("publish - publish the brag report to the server")
439 print_stderr("update-metadata <update-metadata-options> - Update")
440 print_stderr(" ownership information for bragging")
441 print_stderr("clear-metadata - Clear information set by update-metadata")
442 print_stderr("unpublish --yes-i-am-shy - delete the brag report from the server")
445 print_stderr("update-metadata options:")
446 print_stderr("--name= - Name of the cluster")
447 print_stderr("--organization= - Name of the organization")
448 print_stderr("--email= - Email contact address")
449 print_stderr("--description= - Reporting use-case")
450 print_stderr("--url= - The URL that is used to publish and unpublish")
453 def update_metadata():
455 possibles
= ['name', 'organization', 'email', 'description', 'url']
457 #get the existing values
458 info
= get_ownership_info();
460 for index
in range(2, len(sys
.argv
)):
461 mo
= re
.search("--(\S+)=(.*)", sys
.argv
[index
])
472 print_stderr("Unexpect option --" + k
)
476 (rc
, o
, e
) = run_command(['ceph', 'config-key', 'put',
477 CLUSTER_OWNERSHIP_NAME
, str(info
)])
480 def clear_metadata():
481 (rc
, o
, e
) = run_command(['ceph', 'config-key', 'del',
482 CLUSTER_OWNERSHIP_NAME
])
486 data
, url
= output_json()
488 print_stderr("Cannot publish until a URL is set using update-metadata")
492 print_stderr("PUT " + str(url
) + " : " + str(data
))
493 req
= requests
.put(url
, data
=data
)
494 if req
.status_code
!= 201:
495 print_stderr("Failed to publish, server responded with code " + str(req
.status_code
))
496 print_stderr(req
.text
)
502 if len(sys
.argv
) <= 2 or sys
.argv
[2] != '--yes-i-am-shy':
503 print_stderr("unpublish should be followed by --yes-i-am-shy")
507 owner
= get_ownership_info()
516 print_stderr("URL is not updated yet")
521 params
= {'uuid':uuid
}
522 req
= requests
.delete(url
, params
=params
)
523 if req
.status_code
!= 200:
524 print_stderr("Failed to unpublish, server responsed with code " + str(req
.status_code
))
530 if len(sys
.argv
) > 1 and ( sys
.argv
[1] == '--verbose' or sys
.argv
[1] == '-v' ):
534 if len(sys
.argv
) == 1:
535 print(output_json()[0])
537 if sys
.argv
[1] == 'update-metadata':
538 return update_metadata()
539 elif sys
.argv
[1] == 'clear-metadata':
540 return clear_metadata()
541 elif sys
.argv
[1] == 'publish':
543 elif sys
.argv
[1] == 'unpublish':
549 if __name__
== '__main__':