MCPcopy Create free account
hub / github.com/internetarchive/openlibrary / sampledump

Function sampledump

openlibrary/plugins/openlibrary/code.py:162–231  ·  view source on GitHub ↗

Creates a dump of objects from OL database for creating a sample database.

()

Source from the content-addressed store, hash-verified

160
161@infogami.action
162def sampledump():
163 """Creates a dump of objects from OL database for creating a sample database."""
164
165 def expand_keys(keys):
166 def f(k):
167 if isinstance(k, dict):
168 return web.ctx.site.things(k)
169 elif k.endswith("*"):
170 return web.ctx.site.things({"key~": k})
171 else:
172 return [k]
173
174 result = []
175 for k in keys:
176 d = f(k)
177 result += d
178 return result
179
180 def get_references(data, result=None):
181 if result is None:
182 result = []
183
184 if isinstance(data, dict):
185 if "key" in data:
186 result.append(data["key"])
187 else:
188 get_references(data.values(), result)
189 elif isinstance(data, list):
190 for v in data:
191 get_references(v, result)
192 return result
193
194 visiting = {}
195 visited = set()
196
197 def visit(key):
198 if key in visited or key.startswith("/type/"):
199 return
200 elif key in visiting:
201 # This is a case of circular-dependency. Add a stub object to break it.
202 print(json.dumps({"key": key, "type": visiting[key]["type"]}))
203 visited.add(key)
204 return
205
206 thing = web.ctx.site.get(key)
207 if not thing:
208 return
209
210 d = thing.dict()
211 d.pop("permission", None)
212 d.pop("child_permission", None)
213 d.pop("table_of_contents", None)
214
215 visiting[key] = d
216 for ref in get_references(d.values()):
217 visit(ref)
218 visited.add(key)
219

Callers

nothing calls this directly

Calls 4

setFunction · 0.85
expand_keysFunction · 0.85
rangeFunction · 0.85
visitFunction · 0.85

Tested by

no test coverage detected