Context Navigation

forum-sync-script.acc @ 2686

Last change on this file since 2686 was 2686, checked in by mwindhouwer, 11 years ago

Fixed the throtteling:

all SQL request go through the ISOcat.database throttle (settings in the ISOcat config), also the indirect DBX ones
big (more then 100 DCs) DCS to DCIF go through the big-dcif throttle (hardcoded settings)

File size: 16.0 KB

Line
1	<accessor debug="false">
2	<name>ISOcat.access.data.forum.sync.script</name>
3	<descr>Sync ISOcat and the forums user accounts.</descr>
4	<idoc>
5	<instr>
6	<type>ISOcat.access.data.query</type>
7	<query>
8	<update>
9	UPDATE
10	core_user
11	SET
12	sync = true
13	FROM
14	core_group,
15	link_group_members
16	WHERE
17	core_group.forum_group IS NOT NULL
18	AND
19	core_group.sync = true
20	AND
21	link_group_members."group" = core_group.id
22	AND
23	core_user.id = link_group_members.member
24	;
25	</update>
26	</query>
27	<target>var:void</target>
28	</instr>
29	<instr>
30	<type>isocat.sloot.dbx</type>
31	<operator>ffcpl:/dbx/users-sync.xml</operator>
32	<debug>var:accessor-debug</debug>
33	<trace>var:accessor-trace</trace>
34	<target>var:users</target>
35	</instr>
36	<accessor-debug>
37	<instr>
38	<type>copy</type>
39	<operand>var:users</operand>
40	<target>ffcpl:/trace/forum-sync.users.xml</target>
41	</instr>
42	</accessor-debug>
43	<if>
44	<cond>
45	<instr>
46	<type>sloot.xpath2eval</type>
47	<operand>var:users</operand>
48	<operator>
49	<xpath>count(//user)>0</xpath>
50	</operator>
51	<target>this:cond</target>
52	</instr>
53	</cond>
54	<then>
55	<!-- convert into CSV -->
56	<instr>
57	<type>xslt2</type>
58	<operand>var:users</operand>
59	<operator>
60	<xsl:stylesheet
61	version="2.0"
62	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
63	>
64	<xsl:output method="text" encoding="UTF-8"/>
65	<xsl:variable name="NL" select="system-property('line.separator')"/>
66	<xsl:template match="text()"/>
67	<xsl:template match="user">
68	<xsl:value-of select="login"/>
69	<xsl:text>,</xsl:text>
70	<xsl:value-of select="password"/>
71	<xsl:text>,</xsl:text>
72	<xsl:value-of select="email"/>
73	<xsl:text>,</xsl:text>
74	<xsl:value-of select="string-join(group,' ')"/>
75	<xsl:if test="exists(following-sibling::user)">
76	<xsl:value-of select="\$NL"/>
77	</xsl:if>
78	</xsl:template>
79	</xsl:stylesheet>
80	</operator>
81	<target>ffcpl:/store/isocat.csv</target>
82	</instr>
83	<!-- copy CSV to the phpBB forum store directory -->
84	<instr>
85	<type>copy</type>
86	<operand>ffcpl:/store/isocat.csv</operand>
87	<target>$forum_dir$store/isocat.csv</target>
88	</instr>
89	<!-- login to phpBB -->
90	<instr>
91	<type>httpPost</type>
92	<url>$forum_site$ucp.php?mode=login</url>
93	<nvp>
94	<nvp>
95	<username>$forum_user$</username>
96	<password>$forum_password$</password>
97	<login>Login</login>
98	<redirect>./index.php?</redirect>
99	</nvp>
100	</nvp>
101	<target>var:login</target>
102	</instr>
103	<instr>
104	<type>copy</type>
105	<operand>var:login</operand>
106	<target>ffcpl:/trace/forum-sync.login.html</target>
107	</instr>
108	<instr>
109	<type>tagSoup</type>
110	<operand>var:login</operand>
111	<target>var:login</target>
112	</instr>
113	<instr>
114	<type>copy</type>
115	<operand>var:login</operand>
116	<target>ffcpl:/trace/forum-sync.login.xhtml</target>
117	</instr>
118	<!-- get the sid -->
119	<instr>
120	<type>sloot.xpath2eval</type>
121	<operand>var:login</operand>
122	<operator>
123	<xpath>//div[@id='message']//a[contains(@href,'index.php?&sid=')]/substring-after(@href,'$forum_site$index.php?&sid=')</xpath>
124	</operator>
125	<namespace>
126	<namespace>
127	<default>http://www.w3.org/1999/xhtml</default>
128	</namespace>
129	</namespace>
130	<param>
131	<return>
132	<canonical/>
133	<string/>
134	</return>
135	</param>
136	<target>var:sid</target>
137	</instr>
138	<instr>
139	<type>log</type>
140	<operand>var:sid</operand>
141	</instr>
142	<!-- get the phpBB ACP login -->
143	<instr>
144	<type>sloot.xquery</type>
145	<operator>
146	<xquery>
147	declare variable \$sid as node() external;
148	<url>$forum_site$adm/index.php?sid={string(\$sid)}</url>
149	</xquery>
150	</operator>
151	<sid>var:sid</sid>
152	<target>var:url</target>
153	</instr>
154	<instr>
155	<type>httpGet</type>
156	<url>curi:var:url</url>
157	<target>var:login-acp</target>
158	</instr>
159	<instr>
160	<type>copy</type>
161	<operand>var:login-acp</operand>
162	<target>ffcpl:/trace/forum-sync.login-acp.html</target>
163	</instr>
164	<instr>
165	<type>tagSoup</type>
166	<operand>var:login-acp</operand>
167	<target>var:login-acp</target>
168	</instr>
169	<instr>
170	<type>copy</type>
171	<operand>var:login-acp</operand>
172	<target>ffcpl:/trace/forum-sync.login-acp.xhtml</target>
173	</instr>
174	<!-- get the credentials -->
175	<instr>
176	<type>sloot.xpath2eval</type>
177	<operand>var:login-acp</operand>
178	<operator>
179	<xpath>//input[@name='credential']/@value</xpath>
180	</operator>
181	<namespace>
182	<namespace>
183	<default>http://www.w3.org/1999/xhtml</default>
184	</namespace>
185	</namespace>
186	<param>
187	<return>
188	<canonical/>
189	<string/>
190	</return>
191	</param>
192	<target>var:cred</target>
193	</instr>
194	<instr>
195	<type>log</type>
196	<operand>var:cred</operand>
197	</instr>
198	<!-- login to the phpBB ACP -->
199	<instr>
200	<type>sloot.xquery</type>
201	<operator>
202	<xquery>
203	declare variable \$sid as node() external;
204	<url>$forum_site$adm/index.php?sid={string(\$sid)}</url>
205	</xquery>
206	</operator>
207	<sid>var:sid</sid>
208	<target>var:url</target>
209	</instr>
210	<instr>
211	<type>xslt2</type>
212	<operand>
213	<null/>
214	</operand>
215	<operator>
216	<xsl:stylesheet
217	xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
218	version="2.0"
219	>
220	<xsl:param name="sid"/>
221	<xsl:param name="cred"/>
222	<xsl:template match="/">
223	<nvp>
224	<username>$forum_user$</username>
225	<xsl:element name="{concat('password_',\$cred)}">
226	<xsl:text>$forum_password$</xsl:text>
227	</xsl:element>
228	<credential>
229	<xsl:value-of select="\$cred"/>
230	</credential>
231	<login>Login</login>
232	<redirect>
233	<xsl:text>./../adm/index.php?sid=</xsl:text>
234	<xsl:value-of select="\$sid"/>
235	</redirect>
236	</nvp>
237	</xsl:template>
238	</xsl:stylesheet>
239	</operator>
240	<sid>var:sid</sid>
241	<cred>var:cred</cred>
242	<target>var:param</target>
243	</instr>
244	<instr>
245	<type>httpPost</type>
246	<url>curi:var:url</url>
247	<nvp>var:param</nvp>
248	<target>var:acp-login</target>
249	</instr>
250	<instr>
251	<type>copy</type>
252	<operand>var:acp-login</operand>
253	<target>ffcpl:/trace/forum-sync.acp-login.html</target>
254	</instr>
255	<instr>
256	<type>tagSoup</type>
257	<operand>var:acp-login</operand>
258	<target>var:acp-login</target>
259	</instr>
260	<instr>
261	<type>copy</type>
262	<operand>var:acp-login</operand>
263	<target>ffcpl:/trace/forum-sync.acp-login.xhtml</target>
264	</instr>
265	<!-- get the new sid -->
266	<instr>
267	<type>sloot.xpath2eval</type>
268	<operand>var:acp-login</operand>
269	<operator>
270	<xpath>//meta[@http-equiv='refresh']/substring-after(@content,'3;url=$forum_site$adm/index.php?sid=')</xpath>
271	</operator>
272	<namespace>
273	<namespace>
274	<default>http://www.w3.org/1999/xhtml</default>
275	</namespace>
276	</namespace>
277	<param>
278	<return>
279	<canonical/>
280	<string/>
281	</return>
282	</param>
283	<target>var:sid</target>
284	</instr>
285	<instr>
286	<type>log</type>
287	<operand>var:sid</operand>
288	</instr>
289	<!-- get the user import form -->
290	<instr>
291	<type>sloot.xquery</type>
292	<operator>
293	<xquery>
294	declare variable \$sid as node() external;
295	<url>$forum_site$adm/index.php?sid={string(\$sid)}&amp;i=import_users&amp;mode=import</url>
296	</xquery>
297	</operator>
298	<sid>var:sid</sid>
299	<target>var:url</target>
300	</instr>
301	<instr>
302	<type>httpGet</type>
303	<url>curi:var:url</url>
304	<target>var:import-users</target>
305	</instr>
306	<instr>
307	<type>copy</type>
308	<operand>var:import-users</operand>
309	<target>ffcpl:/trace/forum-sync.import-users.html</target>
310	</instr>
311	<instr>
312	<type>tagSoup</type>
313	<operand>var:import-users</operand>
314	<target>var:import-users</target>
315	</instr>
316	<instr>
317	<type>copy</type>
318	<operand>var:import-users</operand>
319	<target>ffcpl:/trace/forum-sync.import-users.xhtml</target>
320	</instr>
321	<!-- get the form tokens -->
322	<instr>
323	<type>sloot.xpath2eval</type>
324	<operand>var:import-users</operand>
325	<operator>
326	<xpath>//input[@name='creation_time']/@value</xpath>
327	</operator>
328	<namespace>
329	<namespace>
330	<default>http://www.w3.org/1999/xhtml</default>
331	</namespace>
332	</namespace>
333	<param>
334	<return>
335	<canonical/>
336	<string/>
337	</return>
338	</param>
339	<target>var:ct</target>
340	</instr>
341	<instr>
342	<type>log</type>
343	<operand>var:ct</operand>
344	</instr>
345	<instr>
346	<type>sloot.xpath2eval</type>
347	<operand>var:import-users</operand>
348	<operator>
349	<xpath>//input[@name='form_token']/@value</xpath>
350	</operator>
351	<namespace>
352	<namespace>
353	<default>http://www.w3.org/1999/xhtml</default>
354	</namespace>
355	</namespace>
356	<param>
357	<return>
358	<canonical/>
359	<string/>
360	</return>
361	</param>
362	<target>var:ft</target>
363	</instr>
364	<instr>
365	<type>log</type>
366	<operand>var:ft</operand>
367	</instr>
368	<!-- now we need to sleep a bit to make phpBB think we're human -->
369	<instr>
370	<type>sleep</type>
371	<operator>
372	<time>1000</time>
373	</operator>
374	</instr>
375	<!-- finally do the actual user import -->
376	<instr>
377	<type>sloot.xquery</type>
378	<operator>
379	<xquery>
380	declare variable \$sid as node() external;
381	<url>$forum_site$adm/index.php?i=import_users&amp;sid={string(\$sid)}&amp;mode=import</url>
382	</xquery>
383	</operator>
384	<sid>var:sid</sid>
385	<target>var:url</target>
386	</instr>
387	<instr>
388	<type>sloot.xquery</type>
389	<operator>
390	<xquery>
391	declare variable \$ct as node() external;
392	declare variable \$ft as node() external;
393	<string>filename=isocat.csv&amp;delim=%2C&amp;enclosure=%22&amp;dateformat=D+M+d%2C+Y+g%3Ai+a&amp;birthdateformat=&amp;email_users=0&amp;pass_fmt=0&amp;required_mapping%5Busername%5D=0&amp;required_mapping%5Buser_password%5D=1&amp;required_mapping%5Buser_email%5D=2&amp;optional_mapping%5Buser_regdate%5D=-1&amp;optional_mapping%5Buser_ip%5D=-1&amp;optional_mapping%5Buser_from%5D=-1&amp;optional_mapping%5Buser_icq%5D=-1&amp;optional_mapping%5Buser_yim%5D=-1&amp;optional_mapping%5Buser_msnm%5D=-1&amp;optional_mapping%5Buser_jabber%5D=-1&amp;optional_mapping%5Buser_occ%5D=-1&amp;optional_mapping%5Buser_interests%5D=-1&amp;optional_mapping%5Buser_birthday%5D=-1&amp;submit=Submit&amp;creation_time={string(\$ct)}&amp;form_token={string(\$ft)}</string>
394	</xquery>
395	</operator>
396	<ct>var:ct</ct>
397	<ft>var:ft</ft>
398	<target>var:arg</target>
399	</instr>
400	<instr>
401	<type>sloot.xpath2eval</type>
402	<operand>var:arg</operand>
403	<operator>
404	<xpath>.</xpath>
405	</operator>
406	<param>
407	<return>
408	<string/>
409	</return>
410	</param>
411	<target>var:arg</target>
412	</instr>
413	<instr>
414	<type>log</type>
415	<operand>var:arg</operand>
416	</instr>
417	<instr>
418	<type>httpPost</type>
419	<url>curi:var:url</url>
420	<!-- order is important here, the first non-url and non-state argument is taken for the POST content -->
421	<arg>var:arg</arg>
422	<headers>
423	<nvp>
424	<Content-Type>application/x-www-form-urlencoded</Content-Type>
425	</nvp>
426	</headers>
427	<config>
428	<config>
429	<timeout>60000</timeout>
430	</config>
431	</config>
432	<target>var:users-import</target>
433	</instr>
434	<instr>
435	<type>copy</type>
436	<operand>var:users-import</operand>
437	<target>ffcpl:/trace/forum-sync.users-import.html</target>
438	</instr>
439	<instr>
440	<type>tagSoup</type>
441	<operand>var:users-import</operand>
442	<target>var:users-import</target>
443	</instr>
444	<instr>
445	<type>copy</type>
446	<operand>var:users-import</operand>
447	<target>ffcpl:/trace/forum-sync.users-import.xhtml</target>
448	</instr>
449	<instr>
450	<type>sloot.xpath2eval</type>
451	<operand>var:users-import</operand>
452	<operator>
453	<xpath>concat('INF: ',//div[@class='successbox']/string-join(p//text(),' '))</xpath>
454	</operator>
455	<namespace>
456	<namespace>
457	<default>http://www.w3.org/1999/xhtml</default>
458	</namespace>
459	</namespace>
460	<param>
461	<return>
462	<canonical/>
463	<string/>
464	</return>
465	</param>
466	<target>var:success</target>
467	</instr>
468	<instr>
469	<type>sloot.log</type>
470	<operand>var:success</operand>
471	</instr>
472	<!-- set the forum account id for the new users -->
473	<instr>
474	<type>xslt2</type>
475	<operand>var:users-import</operand>
476	<operator>
477	<xsl:stylesheet
478	xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0"
479	xmlns:xhtml="http://www.w3.org/1999/xhtml"
480	>
481	<xsl:template match="text()">
482	<xsl:analyze-string select="." regex="^([0-9]+):(.+)\$">
483	<xsl:matching-substring>
484	<sql>
485	<xsl:text>UPDATE core_user SET forum = </xsl:text>
486	<xsl:value-of select="regex-group(1)"/>
487	<xsl:text> WHERE login = '</xsl:text>
488	<xsl:value-of select="replace(regex-group(2),'''','''''')"/>
489	<xsl:text>';</xsl:text>
490	</sql>
491	</xsl:matching-substring>
492	</xsl:analyze-string>
493	</xsl:template>
494	<xsl:template match="/">
495	<batch>
496	<xsl:apply-templates select="//xhtml:div[@class='successbox']"/>
497	</batch>
498	</xsl:template>
499	</xsl:stylesheet>
500	</operator>
501	<target>var:sql</target>
502	</instr>
503	<instr>
504	<type>copy</type>
505	<operand>var:sql</operand>
506	<target>ffcpl:/trace/forum-sync.new-user-sync.sql</target>
507	</instr>
508	<if>
509	<cond>
510	<instr>
511	<type>sloot.xpath2eval</type>
512	<operand>var:sql</operand>
513	<operator>
514	<xpath>exists(/batch/sql)</xpath>
515	</operator>
516	<target>this:cond</target>
517	</instr>
518	</cond>
519	<then>
520	<instr>
521	<type>isocat.sqlBatch</type>
522	<operand>var:sql</operand>
523	<target>var:void</target>
524	</instr>
525	</then>
526	</if>
527	<!-- reset the sync flag of the groups and users -->
528	<instr>
529	<type>ISOcat.access.data.query</type>
530	<query>
531	<batch>
532	<sql>
533	UPDATE core_group SET sync = false WHERE sync = true;
534	</sql>
535	<sql>
536	UPDATE core_user SET sync = false WHERE sync = true;
537	</sql>
538	</batch>
539	</query>
540	<target>var:void</target>
541	</instr>
542	<!-- tell what we did -->
543	<instr>
544	<type>sloot.xquery</type>
545	<operator>
546	<xquery>
547	declare variable \$users as node() external;
548	<msg>INF: {current-dateTime()}:ISOcat.access.data.forum.sync: synced user accounts[{count(\$users//user)}]</msg>
549	</xquery>
550	</operator>
551	<users>var:users</users>
552	<target>var:msg</target>
553	</instr>
554	<instr>
555	<type>sloot.log</type>
556	<operand>var:msg</operand>
557	</instr>
558	</then>
559	<else>
560	<!-- tell that we did nothing -->
561	<instr>
562	<type>sloot.xquery</type>
563	<operator>
564	<xquery>
565	<msg>INF: {current-dateTime()}:ISOcat.access.data.forum.sync: no user accounts to sync</msg>
566	</xquery>
567	</operator>
568	<target>var:msg</target>
569	</instr>
570	<instr>
571	<type>sloot.log</type>
572	<operand>var:msg</operand>
573	</instr>
574	</else>
575	</if>
576	<instr>
577	<type>copy</type>
578	<operand>
579	<null/>
580	</operand>
581	<target>this:response</target>
582	</instr>
583	</idoc>
584	<exception>
585	<debug/>
586	<trace>ffcpl:/trace/forum-sync.script.exception.xml</trace>
587	<throw/>
588	</exception>
589	<response>
590	<debug>ffcpl:/trace/forum-sync.script.response.xml</debug>
591	<expire>now</expire>
592	</response>
593	</accessor>

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: cats/ISOcat/trunk/mod-ISOcat-access-data/access/forum-sync-script.acc @ 2686

Download in other formats: