[metrics-commit] CVS update: MODIFIED: source, packetparser.py ...
Joe Bester
commit at globus.org
Thu Dec 17 13:08:10 CST 2009
User: bester
Date: 09/12/17 13:08:10
Modified: python/tools/source Makefile.am dnscache.py gram5packet.py
packetparser.py usage-tables.sql
Added: python/tools/source myproxypacket.py
Log:
Add handling of MyProxy usage packets from Venkat Yekkirala at NCSA
Fix some python style issues in gram packet and dns cache
Revision Changes Path
1.4 +1 -0 usage/python/tools/source/Makefile.am
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/Makefile.am.diff?r1=1.3&r2=1.4
(In the diff below, changes in quantity of whitespace are not shown.)
Index: Makefile.am
===================================================================
RCS file: /home/globdev/CVS/globus-packages/usage/python/tools/source/Makefile.am,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -b -r1.3 -r1.4
--- Makefile.am 20 Oct 2009 14:09:54 -0000 1.3
+++ Makefile.am 17 Dec 2009 19:08:10 -0000 1.4
@@ -23,6 +23,7 @@
packetparser.py \
rftpacket.py \
rlspacket.py \
+ myproxypacket.py \
usageconfig.py \
usagepacket.py
usage_manual = globus-usage-collector.8 globus-usage-uploader.8
1.3 +1 -1 usage/python/tools/source/dnscache.py
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/dnscache.py.diff?r1=1.2&r2=1.3
(In the diff below, changes in quantity of whitespace are not shown.)
Index: dnscache.py
===================================================================
RCS file: /home/globdev/CVS/globus-packages/usage/python/tools/source/dnscache.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- dnscache.py 20 Oct 2009 14:09:54 -0000 1.2
+++ dnscache.py 17 Dec 2009 19:08:10 -0000 1.3
@@ -18,7 +18,7 @@
import socket
-class DNSCache():
+class DNSCache(object):
"""
DNS Cache
"""
1.3 +106 -16 usage/python/tools/source/gram5packet.py
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/gram5packet.py.diff?r1=1.2&r2=1.3
(In the diff below, changes in quantity of whitespace are not shown.)
Index: gram5packet.py
===================================================================
RCS file: /home/globdev/CVS/globus-packages/usage/python/tools/source/gram5packet.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- gram5packet.py 20 Oct 2009 14:09:54 -0000 1.2
+++ gram5packet.py 17 Dec 2009 19:08:10 -0000 1.3
@@ -69,6 +69,22 @@
CUsagePacket.upload_many(dbclass, cursor, packets)
def get_job_manager_id(self, cursor):
+ """
+ Determine the job manager key which matches the host, version,
+ LRM, and configuration in this packet. If this job manager instance
+ is not defined in the table, attempt to insert it into the
+ gram5_job_managers table and return that id.
+
+ Arguments:
+ self -- A gram5packet.GRAM5Packet object
+ cursor -- An SQL cursor to use if we need to insert this job manager
+ into the table
+
+ Returns:
+ An integer key to the gram5_job_managers table or None if the job
+ manager is not defined or it can't be parsed. As a side effect, this
+ key may be newly defined and cached.
+ """
host_id = self.get_host_id()
version_id = self.get_version_id(cursor)
lrm_id = self.get_lrm_id(cursor)
@@ -95,6 +111,22 @@
return job_manager_id
def get_job_manager_instance_id(self, cursor):
+ """
+ Determine the job manager instance key which matches the job manager
+ start type, host and configuration in this packet. If this job manager
+ instance is not defined in the table, attempt to insert it into the
+ gram5_job_manager_instances table and return that id.
+
+ Arguments:
+ self -- A gram5packet.GRAM5Packet object
+ cursor -- An SQL cursor to use if we need to insert this job manager
+ instance into the table
+
+ Returns:
+ An integer key to the gram5_job_managers table or None if the job
+ manager is not defined or it can't be parsed. As a side effect, this
+ key may be newly defined and cached.
+ """
job_manager_id = self.get_job_manager_id(cursor)
uuid = self.data.get('B')
start_time = GRAM5Packet.db_class.TimestampFromTicks(
@@ -116,6 +148,22 @@
return job_manager_instance_id
def get_job_manager_instance_id_by_uuid(self, cursor):
+ """
+ Determine the job manager instance key which matches the job manager
+ UUID and start time. If this instance is not defined in the table,
+ attempt to insert it into the
+ gram5_job_manager_instances table and return that id.
+
+ Arguments:
+ self -- A gram5packet.GRAM5Packet object
+ cursor -- An SQL cursor to use if we need to insert this job manager
+ instance into the table
+
+ Returns:
+ An integer key to the gram5_job_manager_instances table or None if the
+ job manager is not defined or it can't be parsed. As a side effect,
+ this key may be newly defined and cached.
+ """
uuid = self.data.get('B')
start_time = None
if self.data.get('A') != None:
@@ -160,6 +208,22 @@
self.data.get("HOSTNAME"))
def get_lrm_id(self, cursor):
+ """
+ Determine the LRM key which matches the LRM
+ in this packet. If this LRM is not defined in the cache,
+ attempt to insert it into the gram5_lrms table and return that id.
+
+ Arguments:
+ self -- A gram5packet.GRAM5Packet object
+ cursor -- An SQL cursor to use if we need to insert this hostname into
+ the table
+
+ Returns:
+ An integer key to the dns_cache table or None if the version is
+ not defined or can't be parsed. As a side effect, this
+ key may be newly defined and cached.
+
+ """
lrm_id = None
lrm = self.data.get("E")
@@ -458,8 +522,8 @@
SELECT id, jobtype
FROM gram5_job_types""")
for row in cursor:
- [id, jobtype] = row
- GRAM5Packet.job_type_ids[jobtype] = id
+ [jobtypeid, jobtype] = row
+ GRAM5Packet.job_type_ids[jobtype] = jobtypeid
@staticmethod
def __init_clients(cursor):
@@ -484,9 +548,9 @@
SELECT id, host_id, dn
FROM gram5_client""")
for row in cursor:
- [id, host_id, dn] = row
- values = (host_id, dn)
- GRAM5Packet.clients[values] = id
+ [clientid, host_id, clientdn] = row
+ values = (host_id, clientdn)
+ GRAM5Packet.clients[values] = clientid
@staticmethod
def __init_executables(cursor):
@@ -511,11 +575,22 @@
SELECT id, executable, arguments
FROM gram5_executable""")
for row in cursor:
- [id, executable, arguments] = row
- values = (id, executable, arguments)
- GRAM5Packet.executables[values] = id
+ [exeid, executable, arguments] = row
+ values = (exeid, executable, arguments)
+ GRAM5Packet.executables[values] = exeid
def get_lifetime(self):
+ """
+ Compute the lifetime of the job manager instance as the delta between
+ the job manager start time and the packet send time.
+
+ Arguments:
+ self -- The packet containing timestamp data
+
+ Returns:
+ A string of the form "X seconds" containing the difference between
+ those times as floats.
+ """
start_time = float(self.data.get('A'))
send_time = self.send_time_ticks
return "%f seconds" % (send_time - start_time)
@@ -549,8 +624,8 @@
attribute_list = []
- for (name, id) in GRAM5Packet.__rsl_attributes.items():
- if (bitfield & (2**int(id))) != 0:
+ for (name, rslid) in GRAM5Packet.__rsl_attributes.items():
+ if (bitfield & (2**int(rslid))) != 0:
attribute_list.append(name)
attribute_list.sort()
@@ -766,7 +841,7 @@
executable_id = self.get_executable_id(cursor)
rsl_bitfield = self.get_rsl_bitfield(cursor)
gram5_job_file_info = self.get_file_info(cursor)
- jobtype = self.get_job_type(cursor);
+ jobtype = self.get_job_type(cursor)
values = (
job_manager_id,
@@ -805,7 +880,7 @@
user_dn = self.data.get('9')
if client_ip is not None:
- [address, port] = client_ip.split(':')
+ address = client_ip.split(':')[0]
host_id = GRAM5Packet.dns_cache.get_host_id(address)
values = (host_id, user_dn)
@@ -933,13 +1008,28 @@
return file_info_id
def get_job_type(self, cursor):
+ """
+ Determine the job type key which matches the job type used in this
+ packet. If this job type is not defined in the table, attempt to insert
+ it into the gram5_job_types table and return that id.
+
+ Arguments:
+ self -- A gram5packet.GRAM5Packet object
+ cursor -- An SQL cursor to use if we need to insert this job type into
+ the table
+
+ Returns:
+ An integer key to the gram5_job_types table or None if the job type
+ is not defined or can't be parsed. As a side effect, this
+ key may be newly defined and cached.
+ """
job_type_id = None
job_type = self.data.get('H')
values = (job_type,)
- job_type_id = GRAM5Packet.job_type_ids.get(job_type);
+ job_type_id = GRAM5Packet.job_type_ids.get(job_type)
if job_type_id == None:
cursor.execute('''
1.3 +2 -0 usage/python/tools/source/packetparser.py
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/packetparser.py.diff?r1=1.2&r2=1.3
(In the diff below, changes in quantity of whitespace are not shown.)
Index: packetparser.py
===================================================================
RCS file: /home/globdev/CVS/globus-packages/usage/python/tools/source/packetparser.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- packetparser.py 20 Oct 2009 14:09:54 -0000 1.2
+++ packetparser.py 17 Dec 2009 19:08:10 -0000 1.3
@@ -32,6 +32,7 @@
import drspacket
import mdsaggregatorpacket
import ogsadaipacket
+import myproxypacket
import gram5packet
__packet_classifier = \
@@ -49,6 +50,7 @@
(8,0): mpigpacket.MPIGPacket,
(9,0): drspacket.DRSPacket,
(10,1): ogsadaipacket.OGSADAIPacket,
+ (11,0): myproxypacket.MyProxyPacket,
(20,0): gram5packet.GRAM5JMPacket,
(20,1): gram5packet.GRAM5JobPacket
}
1.3 +20 -0 usage/python/tools/source/usage-tables.sql
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/usage-tables.sql.diff?r1=1.2&r2=1.3
(In the diff below, changes in quantity of whitespace are not shown.)
Index: usage-tables.sql
===================================================================
RCS file: /home/globdev/CVS/globus-packages/usage/python/tools/source/usage-tables.sql,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -b -r1.2 -r1.3
--- usage-tables.sql 20 Oct 2009 14:09:54 -0000 1.2
+++ usage-tables.sql 17 Dec 2009 19:08:10 -0000 1.3
@@ -839,3 +839,23 @@
167 GLOBUS_GRAM_PROTOCOL_ERROR_INVALID_USER_NAME
168 GLOBUS_GRAM_PROTOCOL_ERROR_LAST
\.
+
+CREATE TABLE myproxy_packets(
+ id SERIAL,
+ component_code SMALLINT NOT NULL,
+ version_code SMALLINT NOT NULL,
+ send_time TIMESTAMP,
+ ip_address INET,
+ hostname VARCHAR(64) NOT NULL,
+ myproxy_major_version SMALLINT,
+ myproxy_minor_version SMALLINT,
+ task_code SMALLINT, -- 0=Get, 1=Put, 2=Info, 3=Destroy, 4=ChPasswd, 5=StoreEntCred, 6=RetrEntCred, 7=GetTrustRoots
+ task_return_code BOOLEAN,
+ req_lifetime INTERVAL, -- the LIFETIME= value in the protocol request
+ cred_lifetime INTERVAL, -- the actual lifetime of the credential
+ info_bits BIT VARYING(32), -- Bits 1=PAM, 2=SASL, 3=Credential passphrase, 4=trusted retriever (this is a certificate-based authentication), 5=trusted renewer (this is the certificate authorization in the authorization challenge), 6=pubcookie_used, 7=trustroots_requested, 8=trustroots_requested, 9=ca_used_for_GET
+ client_ip INET,
+ user_name VARCHAR(128),
+ user_dn VARCHAR(128),
+ PRIMARY KEY (id)
+);
1.1 usage/python/tools/source/myproxypacket.py
http://viewcvs.globus.org/viewcvs.cgi/usage/python/tools/source/myproxypacket.py?rev=1.1&content-type=text/x-cvsweb-markup
Index: myproxypacket.py
===================================================================
# Copyright 1999-2009 Board of Trustees of University of Illinois
#
# Based on rlspackets.py:
# Copyright 1999-2009 University of Chicago
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Object definition for processing MyProxy usage packets.
"""
from cusagepacket import CUsagePacket
class MyProxyPacket(CUsagePacket):
"""
MyProxy Usage Packet
"""
insert_statement = '''
INSERT INTO myproxy_packets (
component_code,
version_code,
send_time,
ip_address,
hostname,
myproxy_major_version,
myproxy_minor_version,
task_code,
task_return_code,
req_lifetime,
cred_lifetime,
info_bits,
client_ip,
user_name,
user_dn)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
'''
def values(self, dbclass):
"""
Return a values tuple which matches the parameters in the
class's insert_statement.
Arguments:
self -- A MyProxyPacket object
Returns:
Tuple containing
(component_code, version_code, send_time, ip_address,
hostname, myproxy_major_version, myproxy_minor_version, task_code,
task_return_code, req_lifetime, cred_lifetime, info_bits, client_ip,
user_name, user_dn)
"""
return (
self.component_code,
self.packet_version,
dbclass.Timestamp(*self.send_time),
self.ip_address,
self.data.get('HOSTNAME'),
self.data.get('MAJOR_VER'),
self.data.get('MINOR_VER'),
self.data.get('TASK'),
self.data.get('RET'),
self.data.get('REQ_LTIME'),
self.data.get('CRED_LTIME'),
self.data.get('BITS'),
self.data.get('CLIENTIP'),
self.data.get('USER'),
self.data.get('USERDN'))
More information about the metrics-commit
mailing list